Merge pull request #2133 from unixfox/fix/piratebay-engine

Browse Source 
[fix] piratebay engine
This commit is contained in:
Adam Tauber 2020-08-10 23:04:23 +02:00 committed by GitHub
commit f3bfc4f4a9
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 61 additions and 61 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

119
searx/engines/piratebay.py
View File

@ -1,44 +1,50 @@
# Piratebay (Videos, Music, Files) # Piratebay (Videos, Music, Files)
# #
# @website https://thepiratebay.se # @website https://thepiratebay.org
# @provide-api no (nothing found) # @provide-api yes (https://apibay.org/)
# #
# @using-api no # @using-api yes
# @results HTML (using search portal) # @results JSON
# @stable yes (HTML can change) # @stable no (the API is not documented nor versioned)
# @parse url, title, content, seed, leech, magnetlink # @parse url, title, seed, leech, magnetlink, filesize, publishedDate
from lxml import html from json import loads
from datetime import datetime
from operator import itemgetter from operator import itemgetter
from searx.engines.xpath import extract_text from searx.url_utils import quote
from searx.url_utils import quote, urljoin from searx.utils import get_torrent_size
# engine dependent config # engine dependent config
categories = ['videos', 'music', 'files'] categories = ["videos", "music", "files"]
paging = True
# search-url # search-url
url = 'https://thepiratebay.org/' url = "https://thepiratebay.org/"
search_url = url + 'search/{search_term}/{pageno}/99/{search_type}' search_url = "https://apibay.org/q.php?q={search_term}&cat={search_type}"
# default trackers provided by thepiratebay
trackers = [
"udp://tracker.coppersurfer.tk:6969/announce",
"udp://9.rarbg.to:2920/announce",
"udp://tracker.opentrackr.org:1337",
"udp://tracker.internetwarriors.net:1337/announce",
"udp://tracker.leechers-paradise.org:6969/announce",
"udp://tracker.coppersurfer.tk:6969/announce",
"udp://tracker.pirateparty.gr:6969/announce",
"udp://tracker.cyberia.is:6969/announce",
]
# piratebay specific type-definitions # piratebay specific type-definitions
search_types = {'files': '0', search_types = {"files": "0",
'music': '100', "music": "100",
'videos': '200'} "videos": "200"}
# specific xpath variables
magnet_xpath = './/a[@title="Download this torrent using magnet"]'
torrent_xpath = './/a[@title="Download this torrent"]'
content_xpath = './/font[@class="detDesc"]'
# do search-request # do search-request
def request(query, params): def request(query, params):
search_type = search_types.get(params['category'], '0') search_type = search_types.get(params["category"], "0")
params['url'] = search_url.format(search_term=quote(query), params["url"] = search_url.format(search_term=quote(query),
search_type=search_type, search_type=search_type)
pageno=params['pageno'] - 1)
return params return params
@ -47,50 +53,43 @@ def request(query, params):
def response(resp): def response(resp):
results = [] results = []
dom = html.fromstring(resp.text) search_res = loads(resp.text)
search_res = dom.xpath('//table[@id="searchResult"]//tr')
# return empty array if nothing is found # return empty array if nothing is found
if not search_res: if search_res[0]["name"] == "No results returned":
return [] return []
# parse results # parse results
for result in search_res[1:]: for result in search_res:
link = result.xpath('.//div[@class="detName"]//a')[0] link = url + "description.php?id=" + result["id"]
href = urljoin(url, link.attrib.get('href')) magnetlink = "magnet:?xt=urn:btih:" + result["info_hash"] + "&dn=" + result["name"]
title = extract_text(link) + "&tr=" + "&tr=".join(trackers)
content = extract_text(result.xpath(content_xpath))
seed, leech = result.xpath('.//td[@align="right"]/text()')[:2]
# convert seed to int if possible params = {
if seed.isdigit(): "url": link,
seed = int(seed) "title": result["name"],
else: "seed": result["seeders"],
seed = 0 "leech": result["leechers"],
"magnetlink": magnetlink,
"template": "torrent.html"
}
# convert leech to int if possible # extract and convert creation date
if leech.isdigit(): try:
leech = int(leech) date = datetime.fromtimestamp(result.added)
else: params['publishedDate'] = date
leech = 0 except:
pass
magnetlink = result.xpath(magnet_xpath)[0] # let's try to calculate the torrent size
torrentfile_links = result.xpath(torrent_xpath) try:
if torrentfile_links: filesize = get_torrent_size(result["size"], "B")
torrentfile_link = torrentfile_links[0].attrib.get('href') params['filesize'] = filesize
else: except:
torrentfile_link = None pass
# append result # append result
results.append({'url': href, results.append(params)
'title': title,
'content': content,
'seed': seed,
'leech': leech,
'magnetlink': magnetlink.attrib.get('href'),
'torrentfile': torrentfile_link,
'template': 'torrent.html'})
# return results sorted by seeder # return results sorted by seeder
return sorted(results, key=itemgetter('seed'), reverse=True) return sorted(results, key=itemgetter("seed"), reverse=True)

3
searx/settings.yml
View File

@ -544,7 +544,8 @@ engines:
- name : piratebay - name : piratebay
engine : piratebay engine : piratebay
shortcut : tpb shortcut : tpb
url: https://pirateproxy.red/ # You may need to change this URL to a proxy if piratebay is blocked in your country
url: https://thepiratebay.org/
timeout : 3.0 timeout : 3.0
- name : pubmed - name : pubmed