Merge pull request #677 from pydo/feature/seedpeer-engine-integration

Browse Source 
Feature/seedpeer engine integration
This commit is contained in:
Adam Tauber 2016-10-03 13:21:12 +02:00 committed by GitHub
commit e7d005c621
5 changed files with 246 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
AUTHORS.rst
View File

@ -58,3 +58,4 @@ generally made searx better:
- marc @a01200356 - marc @a01200356
- Harry Wood @harry-wood - Harry Wood @harry-wood
- Thomas Renard @threnard - Thomas Renard @threnard
- Pydo `<https://github.com/pydo>`_

78
searx/engines/seedpeer.py Normal file
View File

@ -0,0 +1,78 @@
# Seedpeer (Videos, Music, Files)
#
# @website http://seedpeer.eu
# @provide-api no (nothing found)
#
# @using-api no
# @results HTML (using search portal)
# @stable yes (HTML can change)
# @parse url, title, content, seed, leech, magnetlink
from urlparse import urljoin
from cgi import escape
from urllib import quote
from lxml import html
from operator import itemgetter
from searx.engines.xpath import extract_text
url = 'http://www.seedpeer.eu/'
search_url = url + 'search/{search_term}/7/{page_no}.html'
# specific xpath variables
torrent_xpath = '//*[@id="body"]/center/center/table[2]/tr/td/a'
alternative_torrent_xpath = '//*[@id="body"]/center/center/table[1]/tr/td/a'
title_xpath = '//*[@id="body"]/center/center/table[2]/tr/td/a/text()'
alternative_title_xpath = '//*[@id="body"]/center/center/table/tr/td/a'
seeds_xpath = '//*[@id="body"]/center/center/table[2]/tr/td[4]/font/text()'
alternative_seeds_xpath = '//*[@id="body"]/center/center/table/tr/td[4]/font/text()'
peers_xpath = '//*[@id="body"]/center/center/table[2]/tr/td[5]/font/text()'
alternative_peers_xpath = '//*[@id="body"]/center/center/table/tr/td[5]/font/text()'
age_xpath = '//*[@id="body"]/center/center/table[2]/tr/td[2]/text()'
alternative_age_xpath = '//*[@id="body"]/center/center/table/tr/td[2]/text()'
size_xpath = '//*[@id="body"]/center/center/table[2]/tr/td[3]/text()'
alternative_size_xpath = '//*[@id="body"]/center/center/table/tr/td[3]/text()'
# do search-request
def request(query, params):
params['url'] = search_url.format(search_term=quote(query),
page_no=params['pageno'] - 1)
return params
# get response from search-request
def response(resp):
results = []
dom = html.fromstring(resp.text)
torrent_links = dom.xpath(torrent_xpath)
if len(torrent_links) > 0:
seeds = dom.xpath(seeds_xpath)
peers = dom.xpath(peers_xpath)
titles = dom.xpath(title_xpath)
sizes = dom.xpath(size_xpath)
ages = dom.xpath(age_xpath)
else: # under ~5 results uses a different xpath
torrent_links = dom.xpath(alternative_torrent_xpath)
seeds = dom.xpath(alternative_seeds_xpath)
peers = dom.xpath(alternative_peers_xpath)
titles = dom.xpath(alternative_title_xpath)
sizes = dom.xpath(alternative_size_xpath)
ages = dom.xpath(alternative_age_xpath)
# return empty array if nothing is found
if not torrent_links:
return []
# parse results
for index, result in enumerate(torrent_links):
link = result.attrib.get('href')
href = urljoin(url, link)
results.append({'url': href,
'title': titles[index].text_content(),
'content': '{}, {}'.format(sizes[index], ages[index]),
'seed': seeds[index],
'leech': peers[index],
'template': 'torrent.html'})
# return results sorted by seeder
return sorted(results, key=itemgetter('seed'), reverse=True)

6
searx/settings.yml
View File

@ -495,6 +495,12 @@ engines:
timeout: 6.0 timeout: 6.0
categories : science categories : science
- name : seedpeer
engine : seedpeer
shortcut: speu
categories: files, music, videos
disabled: True
- name : dictzone - name : dictzone
engine : dictzone engine : dictzone
shortcut : dc shortcut : dc

110
tests/unit/engines/seedpeer_fixture.html Normal file
View File

@ -0,0 +1,110 @@
<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
<html xmlns="http://www.w3.org/1999/xhtml">
<head>
</head>
<body>
<div id="header">
<div id="whoIsYou">
<a href="/lang.php"><small>SeedPeer in your own language?</small></a>&nbsp;&nbsp;&nbsp;<a href="http://www.seedpeer.eu"><img src="/images/flags/uk.gif" width="16px" alt="Torrents EN" /></a> <a href="http://spanish.seedpeer.eu"><img src="/images/flags/es.gif" width="16px" alt="Torrents ES" /></a> <a href="http://german.seedpeer.eu"><img src="/images/flags/de.gif" width="16px" alt="Torrents DE" /></a> <a href="http://french.seedpeer.eu"><img src="/images/flags/fr.gif" width="16px" alt="Torrents FR" /></a> <a href="http://portuguese.seedpeer.eu"><img src="/images/flags/pt.gif" width="16px" alt="Torrents Portuguese" /></a> <a href="http://swedish.seedpeer.eu"><img src="/images/flags/se.gif" width="16px" alt="Torrents Sweden" /></a>
</div>
<script type="text/javascript">
whoIsYou();
</script>
<div id="search">
<form action="/search.php" method="get">
<input id="topsearchbar" name="search" value="narcos season 2" />
<input type="submit" class="searchbutton" value="Torrents" />
<input style="color:#000" type="submit" class="searchbutton" name="usenet" value="Usenet Binaries" />
</form>
<div id="suggestion"></div>
</div>
<div id="logo"><a href="/"><img src="/images/logo2.gif" alt="Seedpeer homepage" width="415" height="143" /></a></div>
<div id="subtext"><a href="/">Home</a> &gt; <a href="/search.html">Torrent search</a> &gt; Narcos season 2 | page 1</div>
</div>
<div id="nav">
<ul>
<!--
<li><font style="color:red;font-size:9px;font-weight:bold;">NEW</font><a title="Download TOP Games for FREE" rel="nofollow" href="http://www.bigrebelads.com/affiliate/index?ref=9301" target="_blank">FREE Games</a></li>
-->
<li style="border-left:none" id="categories"><a title="Browse Torrent Categories" href="/browse.html">Categories</a>
<ul>
<li><a title="Browse Anime Torrents" href="/browse.html#6">Anime</a></li>
<li><a title="Browse Game Torrents" href="/browse.html#4">Games</a></li>
<li><a title="Browse Movie Torrents" href="/browse.html#1">Movies</a></li>
<li><a title="Browse Music Torrents" href="/browse.html#3">Music</a></li>
<li><a title="Browse Software Torrents" href="/browse.html#5">Software</a></li>
<li><a title="Browse TV Torrents" href="/browse.html#2">TV Shows</a></li>
<li><a title="Browse Other Torrents" href="/browse.html#7">Others</a></li>
</ul>
</li>
<li><a title="Upload A Torrents" href="/upload.html">Upload torrent</a></li>
<li id="verified"><a title="Verified Torrents" href="/verified.html">Verified</a></li>
<li id="searchoptions"><a title="Search Torrents" href="/search.html">Torrent search</a></li>
<li id="newsgroups"><a style="color:#212b3e" title="News Groups" href="/usenet.html">Usenet Binaries</a></li>
<li id="about" style="border-right:none"><a rel="nofollow" href="/faq.html">About Us</a>
<ul>
<li><a title="SeedPeer Statistics" href="/stats.html">Statistics</a></li>
<li><a title="Contact Us" href="/contact.html">Contact</a></li>
<li><a title="Frequently Asked Questions" href="/faq.html">FAQ</a></li>
<li><a title="SeedPeer API" href="http://api.seedpeer.eu">Our API</a></li>
<li><a title="SeedPeer Blog" href="/blog">Blog</a></li>
</ul>
</li>
<!--<li><a href="/toolbar.php">Our Toolbar</a></li>-->
</ul>
<div class="clear"></div>
</div>
<div id="body"><div id="pageTop"></div>
<div id="headerbox"><h1>Verified <font class="colored">Narcos season 2</font> torrents</h1></div><table width="100%"><tr><th>
<span style="float:right">
<a href="/search/narcos-season-2/8/1.html"><img style="vertical-align:middle" src="/images/comments.gif" alt="comments" /></a> |
<a href="/search/narcos-season-2/7/1.html"><img style="vertical-align:middle" src="/images/ver.gif" alt="verified" /></a>
</span>
<a href="/search/narcos-season-2/1/1.html">Torrent name</a></th><th class="right"><a href="/search/narcos-season-2/2/1.html">Age</a></th><th class="right"><a href="/search/narcos-season-2/3/1.html">Size</a></th><th class="right"><a href="/search/narcos-season-2/4/1.html">Seeds</a></th><th class="right"><a href="/search/narcos-season-2/5/1.html">Peers</a></th><th class="center"><a href="/search/narcos-season-2/6/1.html">Health</a></th><td class="tableAd" rowspan="6"><iframe src="http://creative.wwwpromoter.com/13689?d=300x250" width="300" height="250" style="border: none;" frameborder="0" scrolling="no"></iframe></td></tr><tr class=""><td><a class="pblink" id="pblink_table_item_1" href="" data-tad="431726" data-last-search="narcos+season+2" target="_blank" rel="nofollow"><strong class='colored'>Narcos season 2</strong> Full Version</a></td><td class="right">20 hours</td><td class="right">681.3 MB</td><td class="right"><font color="green">28</font> </td><td class="right"><font color="navy">654</font> </td><td class="center"><img src="/images/h5.gif" alt="Health" /></td></tr><tr class="tdark"><td><a class="pblink" id="pblink_table_item_2" href="" data-tad="431727" data-url="narcos+season+2" target="_blank" rel="nofollow"><strong class='colored'>Narcos season 2</strong> Trusted Source</a></td><td class="right">12 hours</td><td class="right">787.1 MB</td><td class="right"><font color="green">64</font> </td><td class="right"><font color="navy">220</font> </td><td class="center"><img src="/images/h5.gif" alt="Health" /></td></tr><tr class=""><td><a class="pblink" id="pblink_table_item_3" href="" data-tad="431729" data-last-search="narcos+season+2" target="_blank" rel="nofollow"><strong class='colored'>Full Narcos season 2 Download</strong></a> <small><a class="pblink" id="pblink_table_item_4" href="" data-tad="431729" data-last-search="narcos+season+2" target="_blank" rel="nofollow">Usenet</a></small></td><td class="right">24 hours</td><td class="right">775.5 MB</td><td class="right"><font color="green">60</font> </td><td class="right"><font color="navy">236</font> </td><td class="center"><img src="/images/h5.gif" alt="Health" /></td></tr><tr class="tdark"><td><a class="pblink" id="pblink_table_item_5" href="" data-tad="431730" data-last-search="narcos+season+2" target="_blank" rel="nofollow"><strong class='colored'>Narcos season 2</strong> 2014 - DIRECT STREAMING</a> <small><a class="pblink" id="pblink_table_item_6" href="" data-tad="431729" data-last-search="narcos+season+2" target="_blank" rel="nofollow">Movies</a></small></td><td class="right">17 hours</td><td class="right">654.1 MB</td><td class="right"><font color="green">2</font> </td><td class="right"><font color="navy">391</font> </td><td class="center"><img src="/images/h5.gif" alt="Health" /></td></tr><tr class=""><td><a class="pblink" id="pblink_table_item_7" href="" data-tad="431731" data-last-search="narcos+season+2" target="_blank" rel="nofollow"><strong class='colored'>Narcos season 2</strong> 2014</a> <small><a class="pblink" id="pblink_table_item_8" href="" data-tad="431729" data-last-search="narcos+season+2" target="_blank" rel="nofollow">Movies</a></small></td><td class="right">20 hours</td><td class="right">754.5 MB</td><td class="right"><font color="green">21</font> </td><td class="right"><font color="navy">919</font> </td><td class="center"><img src="/images/h5.gif" alt="Health" /></td></tr></table><br /><br /><center><iframe src='http://creative.wwwpromoter.com/13689?d=728x90' width='728' height='90' style='border: none;' frameborder='0' scrolling='no'></iframe><center><span style="float:right;margin:1em .2em 0 0"><a title="Download at the speed of your connection" href="/usenet.php?search=narcos+season+2"><img src="/images/dlf.gif" alt="Search Binaries" /></a></span><div style="margin-bottom:1em;margin-right:290px" id="headerbox"><h1><a href="/searchfeed/narcos+season+2.xml" target="_blank" title="SeedPeer RSS Torrent Search Feed fornarcos season 2"><img src="/images/feedIcon.png" border="0" /></a>&nbsp;2 <font class="colored">Narcos season 2</font> Torrents were found</h1></div><table width="100%"><tr><th>
<span style="float:right">
<a href="/search/narcos-season-2/8/1.html"><img style="vertical-align:middle" src="/images/comments.gif" alt="comments" /></a> |
<a href="/search/narcos-season-2/7/1.html"><img style="vertical-align:middle" src="/images/ver.gif" alt="verified" /></a>
</span>
<a href="/search/narcos-season-2/1/1.html">Torrent name</a></th><th class="right"><a href="/search/narcos-season-2/2/1.html">Age</a></th><th class="right"><a href="/search/narcos-season-2/3/1.html">Size</a></th><th class="right"><a href="/search/narcos-season-2/4/1.html">Seeds</a></th><th class="right"><a href="/search/narcos-season-2/5/1.html">Peers</a></th><th class="center"><a href="/search/narcos-season-2/6/1.html">Health</a></th></tr><tr class=""><td><small class="comments"><a href="http://www.facebook.com/sharer.php?t=Download%20<strong class='colored'>Narcos</strong> <strong class='colored'>Season</strong> <strong class='colored'>2</strong> Complete 7<strong class='colored'>2</strong>0p WebRip EN-SUB x<strong class='colored'>2</strong>64-[MULVAcoded] S0<strong class='colored'>2</strong>%20 torrent&u=http://seedpeer.seedpeer.eu/details/11686840/Narcos-Season-2-Complete-720p-WebRip-EN-SUB-x264-[MULVAcoded]-S02.html"><img src="/images/facebook.png" alt="Add to Facebook" width="14" height="14" /></a></small><a href="/details/11686840/Narcos-Season-2-Complete-720p-WebRip-EN-SUB-x264-[MULVAcoded]-S02.html"><strong class='colored'>Narcos</strong> <strong class='colored'>Season</strong> <strong class='colored'>2</strong> Complete 7<strong class='colored'>2</strong>0p WebRip EN-SUB x<strong class='colored'>2</strong>64-[MULVAcoded] S0<strong class='colored'>2</strong> <small><a href="/browse.html#11686840"></a></small></a></td><td class="right">19 hours</td><td class="right">4.39 GB</td><td class="right"><font color="green">715</font> </td><td class="right"><font color="navy">183</font> </td><td class="center"><img src="/images/h5.gif" alt="Health" width="40" height="11" /></td></tr><tr class="tdark"><td><small class="comments"><a href="http://www.facebook.com/sharer.php?t=Download%20<strong class='colored'>Narcos</strong> - <strong class='colored'>Season</strong> <strong class='colored'>2</strong> - 7<strong class='colored'>2</strong>0p WEBRiP - x<strong class='colored'>2</strong>65 HEVC - ShAaNiG%20 torrent&u=http://seedpeer.seedpeer.eu/details/11685972/Narcos---Season-2---720p-WEBRiP---x265-HEVC---ShAaNiG.html"><img src="/images/facebook.png" alt="Add to Facebook" width="14" height="14" /></a></small><a href="/details/11685972/Narcos---Season-2---720p-WEBRiP---x265-HEVC---ShAaNiG.html"><strong class='colored'>Narcos</strong> - <strong class='colored'>Season</strong> <strong class='colored'>2</strong> - 7<strong class='colored'>2</strong>0p WEBRiP - x<strong class='colored'>2</strong>65 HEVC - ShAaNiG <small><a href="/browse.html#11685972"></a></small></a></td><td class="right">1 day</td><td class="right">2.48 GB</td><td class="right"><font color="green">861</font> </td><td class="right"><font color="navy">332</font> </td><td class="center"><img src="/images/h5.gif" alt="Health" width="40" height="11" /></td></tr></table><div id="headerbox"><h1>Related searches for: <font class="colored">Narcos season 2</font></h1></div><div id="search_suggestions"><br />Other suggested searches: </div><br /><a href="http://torrentz2.eu/search?f=narcos-season-2">Search for "narcos-season-2" on Torrentz2.eu</a><br /><a href="http://torrent-finder.info/show.php?q=narcos-season-2">Search for "narcos-season-2" on Torrent-Finder</a><br /><center><iframe src='http://creative.wwwpromoter.com/13689?d=300x250' width='300' height='250' style='border: none;' frameborder='0' scrolling='no'></iframe>&nbsp;<iframe src='http://creative.wwwpromoter.com/13689?d=300x250' width='300' height='250' style='border: none;' frameborder='0' scrolling='no'></iframe>&nbsp;<iframe src='http://creative.wwwpromoter.com/13689?d=300x250' width='300' height='250' style='border: none;' frameborder='0' scrolling='no'></iframe></center><div id="footer">
<table width="100%">
<tr>
<td width="30%">
<h2>Torrents Download</h2>
<a href="/">Torrent search</a><br />
<a href="/browse.html">Browse categories</a><br />
<a href="/verified.html">Verified Torrents</a><br />
<a href="/order-date.html">Today's torrents</a><br />
<a href="/yesterday.html">Yesterday's torrents</a><br />
<a href="/stats.html">Statistics</a><br />
<br />
<a href="/faq.html#copyright"><strong>Copyright & Removal</strong></a>
</td>
<td width="30%"><h2>Cool Stuff</h2>
<a href="/promotional.php">Promotional</a><br />
<a href="/contact.html">Advertising Information</a><br />
<strong><a href="/plugins.php" title="Add a search plugin to Firefox or Internet Explorer">Search Plugin <span style="color:red">*</span></a></strong><br />
<a href="http://www.utorrent.com">&micro;Torrent Client</a><br />
<a href="/blog">Seedpeer Blog</a><br />
</td>
<td width="30%"><h2>Links</h2>
<a href="http://www.sumotorrent.com" target="_blank"><strong>SumoTorrent</strong></a><br />
<a href="http://www.torrent-finder.info" target="_blank"><strong>Torrent Finder</strong></a><br />
<a href="http://www.torrentpond.com" target="_blank"><strong>TorrentPond</strong></a><br />
<a href="https://www.limetorrents.cc" target="_blank">LimeTorrents.cc</a><br />
<a href="http://www.torrents.to/" target="_blank">Torrents.to</a><br />
<a href="http://www.torrentfunk.com" target="_blank">TorrentFunk</a><br />
<a href="https://monova.org" target="_blank">Monova</a><br />
<a href="http://www.torrentroom.com" target="_blank">TorrentRoom</a><br />
<a href="http://www.katcr.co/" target="_blank">Kickass Torrents Community</a><br />
</td>
<td width="10%"><div id="bottomlogo"></div></td>
</tr>
</table>
<br />
<br />
</div>
</div>
</body>
</html>

51
tests/unit/engines/test_seedpeer.py Normal file
View File

@ -0,0 +1,51 @@
import mock
from collections import defaultdict
from searx.engines import seedpeer
from searx.testing import SearxTestCase
from datetime import datetime
class TestSeedPeerEngine(SearxTestCase):
html = ''
with open('./tests/unit/engines/seedpeer_fixture.html') as fixture:
html += fixture.read()
def test_request(self):
query = 'test_query'
dicto = defaultdict(dict)
dicto['pageno'] = 1
params = seedpeer.request(query, dicto)
self.assertIn('url', params)
self.assertIn(query, params['url'])
self.assertIn('seedpeer.eu', params['url'])
def test_response_raises_attr_error_on_empty_response(self):
self.assertRaises(AttributeError, seedpeer.response, None)
self.assertRaises(AttributeError, seedpeer.response, [])
self.assertRaises(AttributeError, seedpeer.response, '')
self.assertRaises(AttributeError, seedpeer.response, '[]')
def test_response_returns_empty_list(self):
response = mock.Mock(text='<html></html>')
self.assertEqual(seedpeer.response(response), [])
def test_response_returns_all_results(self):
response = mock.Mock(text=self.html)
results = seedpeer.response(response)
self.assertTrue(isinstance(results, list))
self.assertEqual(len(results), 2)
def test_response_returns_correct_results(self):
response = mock.Mock(text=self.html)
results = seedpeer.response(response)
self.assertEqual(
results[0]['title'], 'Narcos - Season 2 - 720p WEBRiP - x265 HEVC - ShAaNiG '
)
self.assertEqual(
results[0]['url'],
'http://www.seedpeer.eu/details/11685972/Narcos---Season-2---720p-WEBRiP---x265-HEVC---ShAaNiG.html'
)
self.assertEqual(results[0]['content'], '2.48 GB, 1 day')
self.assertEqual(results[0]['seed'], '861')
self.assertEqual(results[0]['leech'], '332')