[enh] version 0.12.0

[fix] fix duckduckgo's offset

.travis.yml

@@ -9,6 +9,7 @@ addons:
 language: python
 python:
   - "2.7"
+  - "3.6"
 before_install:
   - "export DISPLAY=:99.0"
   - "sh -e /etc/init.d/xvfb start"
@@ -24,9 +25,9 @@ script:
   - ./manage.sh styles
   - ./manage.sh grunt_build
   - ./manage.sh tests
-  - ./manage.sh py_test_coverage
 after_success:
-  coveralls
+  - ./manage.sh py_test_coverage
+  - coveralls
 notifications:
   irc:
     channels:

AUTHORS.rst

@@ -8,6 +8,7 @@ Major contributing authors:
 - Alexandre Flament `@dalf <https://github.com/dalf>`_
 - @Cqoicebordel
 - Noémi Ványi
+- Marc Abonce Seguin @a01200356
 
 People who have submitted patches/translates, reported bugs, consulted features or
 generally made searx better:
@@ -43,7 +44,6 @@ generally made searx better:
 - Kang-min Liu
 - Kirill Isakov
 - Guilhem Bonnefille
-- Marc Abonce Seguin @a01200356
 - @jibe-b
 - Christian Pietsch @pietsch
 - @Maxqia
@@ -63,3 +63,7 @@ generally made searx better:
 - @firebovine
 - Lorenzo J. Lucchini @luccoj
 - @eig8phei
+- Joachim Cherqui
+- @maxigas
+- Jannik Winkel @kiney
+- @juanitobananas

CHANGELOG.rst

@@ -1,3 +1,26 @@
+0.12.0 2017.06.04
+=================
+
+- Python3 compatibility
+- New engines
+
+  - 1337x.to (files, music, video)
+  - Semantic Scholar (science)
+  - Library Genesis (general)
+  - Framalibre (IT)
+  - Free Software Directory (IT)
+- More compact result UI (oscar theme)
+- Configurable static content and template path
+- Spelling suggestions
+- Multiple engine fixes (duckduckgo, bing, swisscows, yahoo news, bing news, twitter, bing images)
+- Reduced static image size
+- Docker updates
+- Translation updates
+
+
+Special thanks to `NLNet <https://nlnet.nl>`__ for sponsoring multiple features of this release.
+
+
 0.11.0 2017.01.10
 =================
 

Dockerfile

@@ -1,11 +1,11 @@
-FROM alpine:3.3
+FROM alpine:3.5
 MAINTAINER searx <https://github.com/asciimoo/searx>
 LABEL description "A privacy-respecting, hackable metasearch engine."
 
 ENV BASE_URL=False IMAGE_PROXY=False
 EXPOSE 8888
 WORKDIR /usr/local/searx
-CMD ["/usr/bin/tini","--","/usr/local/searx/run.sh"]
+CMD ["/sbin/tini","--","/usr/local/searx/run.sh"]
 
 RUN adduser -D -h /usr/local/searx -s /bin/sh searx searx \
  && echo '#!/bin/sh' >> run.sh \
@@ -36,7 +36,6 @@ RUN echo "@commuedge http://nl.alpinelinux.org/alpine/edge/community" >> /etc/ap
  && apk del \
     build-base \
     python-dev \
-    py-pip\
     libffi-dev \
     openssl-dev \
     libxslt-dev \

manage.sh

@@ -6,12 +6,12 @@ SEARX_DIR="$BASE_DIR/searx"
 ACTION=$1
 
 update_packages() {
-    pip install --upgrade -r "$BASE_DIR/requirements.txt"
+    pip install -r "$BASE_DIR/requirements.txt"
 }
 
 update_dev_packages() {
     update_packages
-    pip install --upgrade -r "$BASE_DIR/requirements-dev.txt"
+    pip install -r "$BASE_DIR/requirements-dev.txt"
 }
 
 check_geckodriver() {

requirements-dev.txt

@@ -3,8 +3,8 @@ mock==2.0.0
 nose2[coverage-plugin]
 pep8==1.7.0
 plone.testing==5.0.0
-robotframework-selenium2library==1.8.0
-robotsuite==1.7.0
+splinter==0.7.5
 transifex-client==0.12.2
 unittest2==1.1.0
 zope.testrunner==4.5.1
+selenium==3.0.1

requirements.txt

@@ -1,11 +1,10 @@
-certifi==2016.9.26
+certifi==2017.1.23
 flask==0.12
 flask-babel==0.11.1
-lxml==3.7.1
-ndg-httpsclient==0.4.2
-pyasn1==0.1.9
+lxml==3.7.3
+idna==2.5
 pygments==2.1.3
 pyopenssl==16.2.0
-python-dateutil==2.5.3
-pyyaml==3.11
-requests[socks]==2.12.4
+python-dateutil==2.6.0
+pyyaml==3.12
+requests[socks]==2.13.0

searx/__init__.py

@@ -18,7 +18,7 @@ along with searx. If not, see < http://www.gnu.org/licenses/ >.
 import certifi
 import logging
 from os import environ
-from os.path import realpath, dirname, join, abspath
+from os.path import realpath, dirname, join, abspath, isfile
 from ssl import OPENSSL_VERSION_INFO, OPENSSL_VERSION
 try:
     from yaml import load
@@ -30,13 +30,24 @@ except:
 searx_dir = abspath(dirname(__file__))
 engine_dir = dirname(realpath(__file__))
 
-# if possible set path to settings using the
-# enviroment variable SEARX_SETTINGS_PATH
+
+def check_settings_yml(file_name):
+    if isfile(file_name):
+        return file_name
+    else:
+        return None
+
+# find location of settings.yml
 if 'SEARX_SETTINGS_PATH' in environ:
-    settings_path = environ['SEARX_SETTINGS_PATH']
-# otherwise using default path
+    # if possible set path to settings using the
+    # enviroment variable SEARX_SETTINGS_PATH
+    settings_path = check_settings_yml(environ['SEARX_SETTINGS_PATH'])
 else:
-    settings_path = join(searx_dir, 'settings.yml')
+    # if not, get it from searx code base or last solution from /etc/searx
+    settings_path = check_settings_yml(join(searx_dir, 'settings.yml')) or check_settings_yml('/etc/searx/settings.yml')
+
+if not settings_path:
+    raise Exception('settings.yml not found')
 
 # load settings
 with open(settings_path) as settings_yaml:
@@ -67,7 +78,7 @@ else:
     logging.basicConfig(level=logging.WARNING)
 
 logger = logging.getLogger('searx')
-
+logger.debug('read configuration from %s', settings_path)
 # Workaround for openssl versions <1.0.2
 # https://github.com/certifi/python-certifi/issues/26
 if OPENSSL_VERSION_INFO[0:3] < (1, 0, 2):

searx/answerers/__init__.py

@@ -1,8 +1,12 @@
 from os import listdir
 from os.path import realpath, dirname, join, isdir
+from sys import version_info
 from searx.utils import load_module
 from collections import defaultdict
 
+if version_info[0] == 3:
+    unicode = str
+
 
 answerers_dir = dirname(realpath(__file__))
 
@@ -10,7 +14,7 @@ answerers_dir = dirname(realpath(__file__))
 def load_answerers():
     answerers = []
     for filename in listdir(answerers_dir):
-        if not isdir(join(answerers_dir, filename)):
+        if not isdir(join(answerers_dir, filename)) or filename.startswith('_'):
             continue
         module = load_module('answerer.py', join(answerers_dir, filename))
         if not hasattr(module, 'keywords') or not isinstance(module.keywords, tuple) or not len(module.keywords):
@@ -30,12 +34,12 @@ def get_answerers_by_keywords(answerers):
 
 def ask(query):
     results = []
-    query_parts = filter(None, query.query.split())
+    query_parts = list(filter(None, query.query.split()))
 
-    if query_parts[0] not in answerers_by_keywords:
+    if query_parts[0].decode('utf-8') not in answerers_by_keywords:
         return results
 
-    for answerer in answerers_by_keywords[query_parts[0]]:
+    for answerer in answerers_by_keywords[query_parts[0].decode('utf-8')]:
         result = answerer(query)
         if result:
             results.append(result)

searx/answerers/random/answerer.py

@@ -1,5 +1,6 @@
 import random
 import string
+import sys
 from flask_babel import gettext
 
 # required answerer attribute
@@ -8,7 +9,11 @@ keywords = ('random',)
 
 random_int_max = 2**31
 
-random_string_letters = string.lowercase + string.digits + string.uppercase
+if sys.version_info[0] == 2:
+    random_string_letters = string.lowercase + string.digits + string.uppercase
+else:
+    unicode = str
+    random_string_letters = string.ascii_lowercase + string.digits + string.ascii_uppercase
 
 
 def random_string():
@@ -24,9 +29,9 @@ def random_int():
     return unicode(random.randint(-random_int_max, random_int_max))
 
 
-random_types = {u'string': random_string,
-                u'int': random_int,
-                u'float': random_float}
+random_types = {b'string': random_string,
+                b'int': random_int,
+                b'float': random_float}
 
 
 # required answerer function

searx/answerers/statistics/answerer.py

@@ -1,8 +1,12 @@
+from sys import version_info
 from functools import reduce
 from operator import mul
 
 from flask_babel import gettext
 
+if version_info[0] == 3:
+    unicode = str
+
 keywords = ('min',
             'max',
             'avg',
@@ -19,22 +23,22 @@ def answer(query):
         return []
 
     try:
-        args = map(float, parts[1:])
+        args = list(map(float, parts[1:]))
     except:
         return []
 
     func = parts[0]
     answer = None
 
-    if func == 'min':
+    if func == b'min':
         answer = min(args)
-    elif func == 'max':
+    elif func == b'max':
         answer = max(args)
-    elif func == 'avg':
+    elif func == b'avg':
         answer = sum(args) / len(args)
-    elif func == 'sum':
+    elif func == b'sum':
         answer = sum(args)
-    elif func == 'prod':
+    elif func == b'prod':
         answer = reduce(mul, args, 1)
 
     if answer is None:

searx/autocomplete.py

@@ -18,7 +18,6 @@ along with searx. If not, see < http://www.gnu.org/licenses/ >.
 
 from lxml import etree
 from json import loads
-from urllib import urlencode
 from searx import settings
 from searx.languages import language_codes
 from searx.engines import (
@@ -26,6 +25,11 @@ from searx.engines import (
 )
 from searx.poolrequests import get as http_get
 
+try:
+    from urllib import urlencode
+except:
+    from urllib.parse import urlencode
+
 
 def get(*args, **kwargs):
     if 'timeout' not in kwargs:

searx/engines/1337x.py

@@ -0,0 +1,39 @@
+from lxml import html
+from searx.engines.xpath import extract_text
+from searx.utils import get_torrent_size
+from searx.url_utils import quote, urljoin
+
+url = 'https://1337x.to/'
+search_url = url + 'search/{search_term}/{pageno}/'
+categories = ['videos']
+paging = True
+
+
+def request(query, params):
+    params['url'] = search_url.format(search_term=quote(query), pageno=params['pageno'])
+
+    return params
+
+
+def response(resp):
+    results = []
+
+    dom = html.fromstring(resp.text)
+
+    for result in dom.xpath('//table[contains(@class, "table-list")]/tbody//tr'):
+        href = urljoin(url, result.xpath('./td[contains(@class, "name")]/a[2]/@href')[0])
+        title = extract_text(result.xpath('./td[contains(@class, "name")]/a[2]'))
+        seed = extract_text(result.xpath('.//td[contains(@class, "seeds")]'))
+        leech = extract_text(result.xpath('.//td[contains(@class, "leeches")]'))
+        filesize_info = extract_text(result.xpath('.//td[contains(@class, "size")]/text()'))
+        filesize, filesize_multiplier = filesize_info.split()
+        filesize = get_torrent_size(filesize, filesize_multiplier)
+
+        results.append({'url': href,
+                        'title': title,
+                        'seed': seed,
+                        'leech': leech,
+                        'filesize': filesize,
+                        'template': 'torrent.html'})
+
+    return results

searx/engines/__init__.py

@@ -72,12 +72,11 @@ def load_engine(engine_data):
             if engine_data['categories'] == 'none':
                 engine.categories = []
             else:
-                engine.categories = map(
-                    str.strip, engine_data['categories'].split(','))
+                engine.categories = list(map(str.strip, engine_data['categories'].split(',')))
             continue
         setattr(engine, param_name, engine_data[param_name])
 
-    for arg_name, arg_value in engine_default_args.iteritems():
+    for arg_name, arg_value in engine_default_args.items():
         if not hasattr(engine, arg_name):
             setattr(engine, arg_name, arg_value)
 
@@ -218,4 +217,5 @@ def get_engines_stats():
 def initialize_engines(engine_list):
     for engine_data in engine_list:
         engine = load_engine(engine_data)
-        engines[engine.name] = engine
+        if engine is not None:
+            engines[engine.name] = engine

searx/engines/archlinux.py

@@ -11,10 +11,9 @@
  @parse        url, title
 """
 
-from urlparse import urljoin
-from urllib import urlencode
 from lxml import html
 from searx.engines.xpath import extract_text
+from searx.url_utils import urlencode, urljoin
 
 # engine dependent config
 categories = ['it']

searx/engines/base.py

@@ -14,10 +14,10 @@
 """
 
 from lxml import etree
-from urllib import urlencode
-from searx.utils import searx_useragent
 from datetime import datetime
 import re
+from searx.url_utils import urlencode
+from searx.utils import searx_useragent
 
 
 categories = ['science']
@@ -73,7 +73,7 @@ def request(query, params):
 def response(resp):
     results = []
 
-    search_results = etree.XML(resp.content)
+    search_results = etree.XML(resp.text)
 
     for entry in search_results.xpath('./result/doc'):
         content = "No description available"

searx/engines/bing.py

@@ -13,9 +13,9 @@
  @todo        publishedDate
 """
 
-from urllib import urlencode
 from lxml import html
 from searx.engines.xpath import extract_text
+from searx.url_utils import urlencode
 
 # engine dependent config
 categories = ['general']
@@ -94,6 +94,8 @@ def _fetch_supported_languages(resp):
     options = dom.xpath('//div[@id="limit-languages"]//input')
     for option in options:
         code = option.xpath('./@id')[0].replace('_', '-')
+        if code == 'nb':
+            code = 'no'
         supported_languages.append(code)
 
     return supported_languages

searx/engines/bing_images.py

@@ -15,11 +15,11 @@
               limited response to 10 images
 """
 
-from urllib import urlencode
 from lxml import html
 from json import loads
 import re
 from searx.engines.bing import _fetch_supported_languages, supported_languages_url
+from searx.url_utils import urlencode
 
 # engine dependent config
 categories = ['images']
@@ -31,7 +31,6 @@ time_range_support = True
 base_url = 'https://www.bing.com/'
 search_string = 'images/search?{query}&count=10&first={offset}'
 time_range_string = '&qft=+filterui:age-lt{interval}'
-thumb_url = "https://www.bing.com/th?id={ihk}"
 time_range_dict = {'day': '1440',
                    'week': '10080',
                    'month': '43200',
@@ -78,30 +77,32 @@ def response(resp):
     dom = html.fromstring(resp.text)
 
     # parse results
-    for result in dom.xpath('//div[@class="dg_u"]/div'):
+    for result in dom.xpath('//div[@id="mmComponent_images_1"]/ul/li/div/div[@class="imgpt"]'):
         link = result.xpath('./a')[0]
 
+        # TODO find actual title
+        title = link.xpath('.//img/@alt')[0]
+
         # parse json-data (it is required to add a space, to make it parsable)
         json_data = loads(_quote_keys_regex.sub(r'\1"\2": \3', link.attrib.get('m')))
 
-        title = link.attrib.get('t1')
-        ihk = link.attrib.get('ihk')
+        url = json_data.get('purl')
+        img_src = json_data.get('murl')
 
-        # url = 'http://' + link.attrib.get('t3')
-        url = json_data.get('surl')
-        img_src = json_data.get('imgurl')
+        thumb_json_data = loads(_quote_keys_regex.sub(r'\1"\2": \3', link.attrib.get('mad')))
+        thumbnail = thumb_json_data.get('turl')
 
         # append result
         results.append({'template': 'images.html',
                         'url': url,
                         'title': title,
                         'content': '',
-                        'thumbnail_src': thumb_url.format(ihk=ihk),
+                        'thumbnail_src': thumbnail,
                         'img_src': img_src})
 
         # TODO stop parsing if 10 images are found
-        if len(results) >= 10:
-            break
+        # if len(results) >= 10:
+        #     break
 
     # return results
     return results

searx/engines/bing_news.py

@@ -11,13 +11,12 @@
  @parse       url, title, content, publishedDate, thumbnail
 """
 
-from urllib import urlencode
-from urlparse import urlparse, parse_qsl
 from datetime import datetime
 from dateutil import parser
 from lxml import etree
 from searx.utils import list_get
 from searx.engines.bing import _fetch_supported_languages, supported_languages_url
+from searx.url_utils import urlencode, urlparse, parse_qsl
 
 # engine dependent config
 categories = ['news']
@@ -113,12 +112,11 @@ def response(resp):
 
         # append result
         if thumbnail is not None:
-            results.append({'template': 'videos.html',
-                            'url': url,
+            results.append({'url': url,
                             'title': title,
                             'publishedDate': publishedDate,
                             'content': content,
-                            'thumbnail': thumbnail})
+                            'img_src': thumbnail})
         else:
             results.append({'url': url,
                             'title': title,

searx/engines/blekko_images.py

@@ -11,7 +11,7 @@
 """
 
 from json import loads
-from urllib import urlencode
+from searx.url_utils import urlencode
 
 # engine dependent config
 categories = ['images']

searx/engines/btdigg.py

@@ -10,11 +10,10 @@
  @parse       url, title, content, seed, leech, magnetlink
 """
 
-from urlparse import urljoin
-from urllib import quote
 from lxml import html
 from operator import itemgetter
 from searx.engines.xpath import extract_text
+from searx.url_utils import quote, urljoin
 from searx.utils import get_torrent_size
 
 # engine dependent config
@@ -38,7 +37,7 @@ def request(query, params):
 def response(resp):
     results = []
 
-    dom = html.fromstring(resp.content)
+    dom = html.fromstring(resp.text)
 
     search_res = dom.xpath('//div[@id="search_res"]/table/tr')
 

searx/engines/currency_convert.py

@@ -1,21 +1,25 @@
-from datetime import datetime
+import json
 import re
 import os
-import json
+import sys
 import unicodedata
 
+from datetime import datetime
+
+if sys.version_info[0] == 3:
+    unicode = str
 
 categories = []
 url = 'https://download.finance.yahoo.com/d/quotes.csv?e=.csv&f=sl1d1t1&s={query}=X'
 weight = 100
 
-parser_re = re.compile(u'.*?(\\d+(?:\\.\\d+)?) ([^.0-9]+) (?:in|to) ([^.0-9]+)', re.I)  # noqa
+parser_re = re.compile(b'.*?(\\d+(?:\\.\\d+)?) ([^.0-9]+) (?:in|to) ([^.0-9]+)', re.I)
 
 db = 1
 
 
 def normalize_name(name):
-    name = name.lower().replace('-', ' ').rstrip('s')
+    name = name.decode('utf-8').lower().replace('-', ' ').rstrip('s')
     name = re.sub(' +', ' ', name)
     return unicodedata.normalize('NFKD', name).lower()
 
@@ -35,7 +39,7 @@ def iso4217_to_name(iso4217, language):
 
 
 def request(query, params):
-    m = parser_re.match(unicode(query, 'utf8'))
+    m = parser_re.match(query)
     if not m:
         # wrong query
         return params

searx/engines/dailymotion.py

@@ -12,10 +12,9 @@
  @todo        set content-parameter with correct data
 """
 
-from urllib import urlencode
 from json import loads
 from datetime import datetime
-from requests import get
+from searx.url_utils import urlencode
 
 # engine dependent config
 categories = ['videos']

searx/engines/deezer.py

@@ -11,7 +11,7 @@
 """
 
 from json import loads
-from urllib import urlencode
+from searx.url_utils import urlencode
 
 # engine dependent config
 categories = ['music']
@@ -30,8 +30,7 @@ embedded_url = '<iframe scrolling="no" frameborder="0" allowTransparency="true"
 def request(query, params):
     offset = (params['pageno'] - 1) * 25
 
-    params['url'] = search_url.format(query=urlencode({'q': query}),
-                                      offset=offset)
+    params['url'] = search_url.format(query=urlencode({'q': query}), offset=offset)
 
     return params
 

searx/engines/deviantart.py

@@ -12,10 +12,10 @@
  @todo        rewrite to api
 """
 
-from urllib import urlencode
 from lxml import html
 import re
 from searx.engines.xpath import extract_text
+from searx.url_utils import urlencode
 
 # engine dependent config
 categories = ['images']

searx/engines/dictzone.py

@@ -10,20 +10,20 @@
 """
 
 import re
-from urlparse import urljoin
 from lxml import html
 from searx.utils import is_valid_lang
+from searx.url_utils import urljoin
 
 categories = ['general']
 url = u'http://dictzone.com/{from_lang}-{to_lang}-dictionary/{query}'
 weight = 100
 
-parser_re = re.compile(u'.*?([a-z]+)-([a-z]+) ([^ ]+)$', re.I)
+parser_re = re.compile(b'.*?([a-z]+)-([a-z]+) ([^ ]+)$', re.I)
 results_xpath = './/table[@id="r"]/tr'
 
 
 def request(query, params):
-    m = parser_re.match(unicode(query, 'utf8'))
+    m = parser_re.match(query)
     if not m:
         return params
 

searx/engines/digbt.py

@@ -10,10 +10,14 @@
  @parse       url, title, content, magnetlink
 """
 
-from urlparse import urljoin
+from sys import version_info
 from lxml import html
 from searx.engines.xpath import extract_text
 from searx.utils import get_torrent_size
+from searx.url_utils import urljoin
+
+if version_info[0] == 3:
+    unicode = str
 
 categories = ['videos', 'music', 'files']
 paging = True
@@ -31,7 +35,7 @@ def request(query, params):
 
 
 def response(resp):
-    dom = html.fromstring(resp.content)
+    dom = html.fromstring(resp.text)
     search_res = dom.xpath('.//td[@class="x-item"]')
 
     if not search_res:

searx/engines/digg.py

@@ -10,10 +10,10 @@
  @parse       url, title, content, publishedDate, thumbnail
 """
 
-from urllib import quote_plus
+from dateutil import parser
 from json import loads
 from lxml import html
-from dateutil import parser
+from searx.url_utils import quote_plus
 
 # engine dependent config
 categories = ['news', 'social media']

searx/engines/doku.py

@@ -9,9 +9,9 @@
 # @stable      yes
 # @parse       (general)    url, title, content
 
-from urllib import urlencode
 from lxml.html import fromstring
 from searx.engines.xpath import extract_text
+from searx.url_utils import urlencode
 
 # engine dependent config
 categories = ['general']  # TODO , 'images', 'music', 'videos', 'files'

searx/engines/duckduckgo.py

@@ -13,11 +13,11 @@
  @todo        rewrite to api
 """
 
-from urllib import urlencode
 from lxml.html import fromstring
 from requests import get
 from json import loads
 from searx.engines.xpath import extract_text
+from searx.url_utils import urlencode
 
 # engine dependent config
 categories = ['general']
@@ -27,7 +27,7 @@ supported_languages_url = 'https://duckduckgo.com/d2030.js'
 time_range_support = True
 
 # search-url
-url = 'https://duckduckgo.com/html?{query}&s={offset}'
+url = 'https://duckduckgo.com/html?{query}&s={offset}&api=/d.js&o=json&dc={dc_param}'
 time_range_url = '&df={range}'
 
 time_range_dict = {'day': 'd',
@@ -41,6 +41,39 @@ title_xpath = './/a[@class="result__a"]'
 content_xpath = './/a[@class="result__snippet"]'
 
 
+# match query's language to a region code that duckduckgo will accept
+def get_region_code(lang):
+    # custom fixes for languages
+    if lang == 'all':
+        region_code = None
+    elif lang[:2] == 'ja':
+        region_code = 'jp-jp'
+    elif lang[:2] == 'sl':
+        region_code = 'sl-sl'
+    elif lang == 'zh-TW':
+        region_code = 'tw-tzh'
+    elif lang == 'zh-HK':
+        region_code = 'hk-tzh'
+    elif lang[-2:] == 'SA':
+        region_code = 'xa-' + lang.split('-')[0]
+    elif lang[-2:] == 'GB':
+        region_code = 'uk-' + lang.split('-')[0]
+    else:
+        region_code = lang.split('-')
+        if len(region_code) == 2:
+            # country code goes first
+            region_code = region_code[1].lower() + '-' + region_code[0].lower()
+        else:
+            # tries to get a country code from language
+            region_code = region_code[0].lower()
+            for lc in supported_languages:
+                lc = lc.split('-')
+                if region_code == lc[0]:
+                    region_code = lc[1].lower() + '-' + lc[0].lower()
+                    break
+    return region_code
+
+
 # do search-request
 def request(query, params):
     if params['time_range'] and params['time_range'] not in time_range_dict:
@@ -48,41 +81,13 @@ def request(query, params):
 
     offset = (params['pageno'] - 1) * 30
 
-    # custom fixes for languages
-    if params['language'] == 'all':
-        locale = None
-    elif params['language'][:2] == 'ja':
-        locale = 'jp-jp'
-    elif params['language'][:2] == 'sl':
-        locale = 'sl-sl'
-    elif params['language'] == 'zh-TW':
-        locale = 'tw-tzh'
-    elif params['language'] == 'zh-HK':
-        locale = 'hk-tzh'
-    elif params['language'][-2:] == 'SA':
-        locale = 'xa-' + params['language'].split('-')[0]
-    elif params['language'][-2:] == 'GB':
-        locale = 'uk-' + params['language'].split('-')[0]
-    else:
-        locale = params['language'].split('-')
-        if len(locale) == 2:
-            # country code goes first
-            locale = locale[1].lower() + '-' + locale[0].lower()
-        else:
-            # tries to get a country code from language
-            locale = locale[0].lower()
-            for lc in supported_languages:
-                lc = lc.split('-')
-                if locale == lc[0]:
-                    locale = lc[1].lower() + '-' + lc[0].lower()
-                    break
-
-    if locale:
+    region_code = get_region_code(params['language'])
+    if region_code:
         params['url'] = url.format(
-            query=urlencode({'q': query, 'kl': locale}), offset=offset)
+            query=urlencode({'q': query, 'kl': region_code}), offset=offset, dc_param=offset)
     else:
         params['url'] = url.format(
-            query=urlencode({'q': query}), offset=offset)
+            query=urlencode({'q': query}), offset=offset, dc_param=offset)
 
     if params['time_range'] in time_range_dict:
         params['url'] += time_range_url.format(range=time_range_dict[params['time_range']])

searx/engines/duckduckgo_definitions.py

@@ -1,10 +1,10 @@
 import json
-from urllib import urlencode
-from re import compile, sub
 from lxml import html
-from searx.utils import html_to_text
+from re import compile
 from searx.engines.xpath import extract_text
 from searx.engines.duckduckgo import _fetch_supported_languages, supported_languages_url
+from searx.url_utils import urlencode
+from searx.utils import html_to_text
 
 url = 'https://api.duckduckgo.com/'\
     + '?{query}&format=json&pretty=0&no_redirect=1&d=1'

searx/engines/duckduckgo_images.py

@@ -0,0 +1,91 @@
+"""
+ DuckDuckGo (Images)
+
+ @website     https://duckduckgo.com/
+ @provide-api yes (https://duckduckgo.com/api),
+              but images are not supported
+
+ @using-api   no
+ @results     JSON (site requires js to get images)
+ @stable      no (JSON can change)
+ @parse       url, title, img_src
+
+ @todo        avoid extra request
+"""
+
+from requests import get
+from json import loads
+from searx.engines.xpath import extract_text
+from searx.engines.duckduckgo import _fetch_supported_languages, supported_languages_url, get_region_code
+from searx.url_utils import urlencode
+
+# engine dependent config
+categories = ['images']
+paging = True
+language_support = True
+safesearch = True
+
+# search-url
+images_url = 'https://duckduckgo.com/i.js?{query}&s={offset}&p={safesearch}&o=json&vqd={vqd}'
+site_url = 'https://duckduckgo.com/?{query}&iar=images&iax=1&ia=images'
+
+
+# run query in site to get vqd number needed for requesting images
+# TODO: find a way to get this number without an extra request (is it a hash of the query?)
+def get_vqd(query):
+    res = get(site_url.format(query=urlencode({'q': query})))
+    content = res.text
+    vqd = content[content.find('vqd=\'') + 5:]
+    vqd = vqd[:vqd.find('\'')]
+    return vqd
+
+
+# do search-request
+def request(query, params):
+    # to avoid running actual external requests when testing
+    if 'is_test' not in params:
+        vqd = get_vqd(query)
+    else:
+        vqd = '12345'
+
+    offset = (params['pageno'] - 1) * 50
+
+    safesearch = params['safesearch'] - 1
+
+    region_code = get_region_code(params['language'])
+    if region_code:
+        params['url'] = images_url.format(
+            query=urlencode({'q': query, 'l': region_code}), offset=offset, safesearch=safesearch, vqd=vqd)
+    else:
+        params['url'] = images_url.format(
+            query=urlencode({'q': query}), offset=offset, safesearch=safesearch, vqd=vqd)
+
+    return params
+
+
+# get response from search-request
+def response(resp):
+    results = []
+
+    content = resp.text
+    try:
+        res_json = loads(content)
+    except:
+        return []
+
+    # parse results
+    for result in res_json['results']:
+        title = result['title']
+        url = result['url']
+        thumbnail = result['thumbnail']
+        image = result['image']
+
+        # append result
+        results.append({'template': 'images.html',
+                        'title': title,
+                        'content': '',
+                        'thumbnail_src': thumbnail,
+                        'img_src': image,
+                        'url': url})
+
+    return results

searx/engines/faroo.py

@@ -10,10 +10,10 @@
  @parse       url, title, content, publishedDate, img_src
 """
 
-from urllib import urlencode
 from json import loads
 import datetime
 from searx.utils import searx_useragent
+from searx.url_utils import urlencode
 
 # engine dependent config
 categories = ['general', 'news']

searx/engines/fdroid.py

@@ -9,9 +9,9 @@
  @parse        url, title, content
 """
 
-from urllib import urlencode
-from searx.engines.xpath import extract_text
 from lxml import html
+from searx.engines.xpath import extract_text
+from searx.url_utils import urlencode
 
 # engine dependent config
 categories = ['files']
@@ -24,8 +24,7 @@ search_url = base_url + 'repository/browse/?{query}'
 
 # do search-request
 def request(query, params):
-    query = urlencode({'fdfilter': query,
-                       'fdpage': params['pageno']})
+    query = urlencode({'fdfilter': query, 'fdpage': params['pageno']})
     params['url'] = search_url.format(query=query)
     return params
 

searx/engines/filecrop.py

@@ -1,5 +1,9 @@
-from urllib import urlencode
-from HTMLParser import HTMLParser
+from searx.url_utils import urlencode
+
+try:
+    from HTMLParser import HTMLParser
+except:
+    from html.parser import HTMLParser
 
 url = 'http://www.filecrop.com/'
 search_url = url + '/search.php?{query}&size_i=0&size_f=100000000&engine_r=1&engine_d=1&engine_e=1&engine_4=1&engine_m=1&pos={index}'  # noqa
@@ -73,8 +77,7 @@ class FilecropResultParser(HTMLParser):
 
 def request(query, params):
     index = 1 + (params['pageno'] - 1) * 30
-    params['url'] = search_url.format(query=urlencode({'w': query}),
-                                      index=index)
+    params['url'] = search_url.format(query=urlencode({'w': query}), index=index)
     return params
 
 

searx/engines/flickr.py

@@ -13,8 +13,8 @@
  More info on api-key : https://www.flickr.com/services/apps/create/
 """
 
-from urllib import urlencode
 from json import loads
+from searx.url_utils import urlencode
 
 categories = ['images']
 

searx/engines/flickr_noapi.py

@@ -12,11 +12,11 @@
  @parse       url, title, thumbnail, img_src
 """
 
-from urllib import urlencode
 from json import loads
 from time import time
 import re
 from searx.engines import logger
+from searx.url_utils import urlencode
 
 
 logger = logger.getChild('flickr-noapi')

searx/engines/framalibre.py

@@ -0,0 +1,69 @@
+"""
+ FramaLibre (It)
+
+ @website     https://framalibre.org/
+ @provide-api no
+
+ @using-api   no
+ @results     HTML
+ @stable      no (HTML can change)
+ @parse       url, title, content, thumbnail, img_src
+"""
+
+from cgi import escape
+from lxml import html
+from searx.engines.xpath import extract_text
+from searx.url_utils import urljoin, urlencode
+
+# engine dependent config
+categories = ['it']
+paging = True
+
+# search-url
+base_url = 'https://framalibre.org/'
+search_url = base_url + 'recherche-par-crit-res?{query}&page={offset}'
+
+# specific xpath variables
+results_xpath = '//div[@class="nodes-list-row"]/div[contains(@typeof,"sioc:Item")]'
+link_xpath = './/h3[@class="node-title"]/a[@href]'
+thumbnail_xpath = './/img[@class="media-object img-responsive"]/@src'
+content_xpath = './/div[@class="content"]//p'
+
+
+# do search-request
+def request(query, params):
+    offset = (params['pageno'] - 1)
+    params['url'] = search_url.format(query=urlencode({'keys': query}),
+                                      offset=offset)
+
+    return params
+
+
+# get response from search-request
+def response(resp):
+    results = []
+
+    dom = html.fromstring(resp.text)
+
+    # parse results
+    for result in dom.xpath(results_xpath):
+        link = result.xpath(link_xpath)[0]
+        href = urljoin(base_url, link.attrib.get('href'))
+        # there's also a span (class="rdf-meta element-hidden" property="dc:title")'s content property for this...
+        title = escape(extract_text(link))
+        thumbnail_tags = result.xpath(thumbnail_xpath)
+        thumbnail = None
+        if len(thumbnail_tags) > 0:
+            thumbnail = extract_text(thumbnail_tags[0])
+            if thumbnail[0] == '/':
+                thumbnail = base_url + thumbnail
+        content = escape(extract_text(result.xpath(content_xpath)))
+
+        # append result
+        results.append({'url': href,
+                        'title': title,
+                        'img_src': thumbnail,
+                        'content': content})
+
+    # return results
+    return results

searx/engines/frinkiac.py

@@ -10,7 +10,7 @@ Frinkiac (Images)
 """
 
 from json import loads
-from urllib import urlencode
+from searx.url_utils import urlencode
 
 categories = ['images']
 

searx/engines/gigablast.py

@@ -11,10 +11,9 @@
 """
 
 from json import loads
-from random import randint
 from time import time
-from urllib import urlencode
 from lxml.html import fromstring
+from searx.url_utils import urlencode
 
 # engine dependent config
 categories = ['general']

searx/engines/github.py

@@ -10,8 +10,8 @@
  @parse       url, title, content
 """
 
-from urllib import urlencode
 from json import loads
+from searx.url_utils import urlencode
 
 # engine dependent config
 categories = ['it']

searx/engines/google.py

@@ -9,11 +9,10 @@
 # @parse       url, title, content, suggestion
 
 import re
-from urllib import urlencode
-from urlparse import urlparse, parse_qsl
 from lxml import html, etree
 from searx.engines.xpath import extract_text, extract_url
-from searx.search import logger
+from searx import logger
+from searx.url_utils import urlencode, urlparse, parse_qsl
 
 logger = logger.getChild('google engine')
 
@@ -112,6 +111,7 @@ title_xpath = './/h3'
 content_xpath = './/span[@class="st"]'
 content_misc_xpath = './/div[@class="f slp"]'
 suggestion_xpath = '//p[@class="_Bmc"]'
+spelling_suggestion_xpath = '//a[@class="spell"]'
 
 # map : detail location
 map_address_xpath = './/div[@class="s"]//table//td[2]/span/text()'
@@ -220,6 +220,12 @@ def response(resp):
     instant_answer = dom.xpath('//div[@id="_vBb"]//text()')
     if instant_answer:
         results.append({'answer': u' '.join(instant_answer)})
+    try:
+        results_num = int(dom.xpath('//div[@id="resultStats"]//text()')[0]
+                          .split()[1].replace(',', ''))
+        results.append({'number_of_results': results_num})
+    except:
+        pass
 
     # parse results
     for result in dom.xpath(results_xpath):
@@ -275,6 +281,9 @@ def response(resp):
         # append suggestion
         results.append({'suggestion': extract_text(suggestion)})
 
+    for correction in dom.xpath(spelling_suggestion_xpath):
+        results.append({'correction': extract_text(correction)})
+
     # return results
     return results
 

searx/engines/google_images.py

@@ -11,9 +11,9 @@
 """
 
 from datetime import date, timedelta
-from urllib import urlencode
 from json import loads
 from lxml import html
+from searx.url_utils import urlencode
 
 
 # engine dependent config

searx/engines/google_news.py

@@ -11,9 +11,8 @@
 """
 
 from lxml import html
-from urllib import urlencode
-from json import loads
 from searx.engines.google import _fetch_supported_languages, supported_languages_url
+from searx.url_utils import urlencode
 
 # search-url
 categories = ['news']

searx/engines/ina.py

@@ -12,11 +12,15 @@
 # @todo        embedded (needs some md5 from video page)
 
 from json import loads
-from urllib import urlencode
 from lxml import html
-from HTMLParser import HTMLParser
-from searx.engines.xpath import extract_text
 from dateutil import parser
+from searx.engines.xpath import extract_text
+from searx.url_utils import urlencode
+
+try:
+    from HTMLParser import HTMLParser
+except:
+    from html.parser import HTMLParser
 
 # engine dependent config
 categories = ['videos']

searx/engines/json_engine.py

@@ -1,11 +1,16 @@
-from urllib import urlencode
-from json import loads
 from collections import Iterable
+from json import loads
+from sys import version_info
+from searx.url_utils import urlencode
+
+if version_info[0] == 3:
+    unicode = str
 
 search_url = None
 url_query = None
 content_query = None
 title_query = None
+paging = False
 suggestion_query = ''
 results_query = ''
 
@@ -20,7 +25,7 @@ first_page_num = 1
 
 def iterate(iterable):
     if type(iterable) == dict:
-        it = iterable.iteritems()
+        it = iterable.items()
 
     else:
         it = enumerate(iterable)

searx/engines/kickass.py

@@ -10,12 +10,11 @@
  @parse       url, title, content, seed, leech, magnetlink
 """
 
-from urlparse import urljoin
-from urllib import quote
 from lxml import html
 from operator import itemgetter
 from searx.engines.xpath import extract_text
 from searx.utils import get_torrent_size, convert_str_to_int
+from searx.url_utils import quote, urljoin
 
 # engine dependent config
 categories = ['videos', 'music', 'files']

searx/engines/mediawiki.py

@@ -14,13 +14,14 @@
 
 from json import loads
 from string import Formatter
-from urllib import urlencode, quote
+from searx.url_utils import urlencode, quote
 
 # engine dependent config
 categories = ['general']
 language_support = True
 paging = True
 number_of_results = 1
+search_type = 'nearmatch'  # possible values: title, text, nearmatch
 
 # search-url
 base_url = 'https://{language}.wikipedia.org/'
@@ -30,7 +31,7 @@ search_postfix = 'w/api.php?action=query'\
     '&format=json'\
     '&sroffset={offset}'\
     '&srlimit={limit}'\
-    '&srwhat=nearmatch'  # search for a near match in the title
+    '&srwhat={searchtype}'
 
 
 # do search-request
@@ -39,7 +40,8 @@ def request(query, params):
 
     string_args = dict(query=urlencode({'srsearch': query}),
                        offset=offset,
-                       limit=number_of_results)
+                       limit=number_of_results,
+                       searchtype=search_type)
 
     format_strings = list(Formatter().parse(base_url))
 

searx/engines/mixcloud.py

@@ -11,8 +11,8 @@
 """
 
 from json import loads
-from urllib import urlencode
 from dateutil import parser
+from searx.url_utils import urlencode
 
 # engine dependent config
 categories = ['music']

searx/engines/nyaa.py

@@ -9,9 +9,9 @@
  @parse        url, title, content, seed, leech, torrentfile
 """
 
-from urllib import urlencode
 from lxml import html
 from searx.engines.xpath import extract_text
+from searx.url_utils import urlencode
 
 # engine dependent config
 categories = ['files', 'images', 'videos', 'music']

searx/engines/openstreetmap.py

@@ -11,7 +11,6 @@
 """
 
 from json import loads
-from searx.utils import searx_useragent
 
 # engine dependent config
 categories = ['map']
@@ -27,9 +26,6 @@ result_base_url = 'https://openstreetmap.org/{osm_type}/{osm_id}'
 def request(query, params):
     params['url'] = base_url + search_string.format(query=query)
 
-    # using searx User-Agent
-    params['headers']['User-Agent'] = searx_useragent()
-
     return params
 
 

searx/engines/photon.py

@@ -10,9 +10,9 @@
  @parse       url, title
 """
 
-from urllib import urlencode
 from json import loads
 from searx.utils import searx_useragent
+from searx.url_utils import urlencode
 
 # engine dependent config
 categories = ['map']

searx/engines/piratebay.py

@@ -8,11 +8,10 @@
 # @stable      yes (HTML can change)
 # @parse       url, title, content, seed, leech, magnetlink
 
-from urlparse import urljoin
-from urllib import quote
 from lxml import html
 from operator import itemgetter
 from searx.engines.xpath import extract_text
+from searx.url_utils import quote, urljoin
 
 # engine dependent config
 categories = ['videos', 'music', 'files']

searx/engines/qwant.py

@@ -12,14 +12,14 @@
 
 from datetime import datetime
 from json import loads
-from urllib import urlencode
-
 from searx.utils import html_to_text
+from searx.url_utils import urlencode
 
 # engine dependent config
 categories = None
 paging = True
 language_support = True
+supported_languages_url = 'https://qwant.com/region'
 
 category_to_keyword = {'general': 'web',
                        'images': 'images',
@@ -46,6 +46,15 @@ def request(query, params):
 
     # add language tag if specified
     if params['language'] != 'all':
+        if params['language'] == 'no' or params['language'].startswith('no-'):
+            params['language'] = params['language'].replace('no', 'nb', 1)
+        if params['language'].find('-') < 0:
+            # tries to get a country code from language
+            for lang in supported_languages:
+                lc = lang.split('-')
+                if params['language'] == lc[0]:
+                    params['language'] = lang
+                    break
         params['url'] += '&locale=' + params['language'].replace('-', '_').lower()
 
     return params
@@ -87,14 +96,45 @@ def response(resp):
                             'thumbnail_src': thumbnail_src,
                             'img_src': img_src})
 
-        elif (category_to_keyword.get(categories[0], '') == 'news' or
-              category_to_keyword.get(categories[0], '') == 'social'):
+        elif category_to_keyword.get(categories[0], '') == 'social':
             published_date = datetime.fromtimestamp(result['date'], None)
-
+            img_src = result.get('img', None)
             results.append({'url': res_url,
                             'title': title,
                             'publishedDate': published_date,
-                            'content': content})
+                            'content': content,
+                            'img_src': img_src})
+
+        elif category_to_keyword.get(categories[0], '') == 'news':
+            published_date = datetime.fromtimestamp(result['date'], None)
+            media = result.get('media', [])
+            if len(media) > 0:
+                img_src = media[0].get('pict', {}).get('url', None)
+            else:
+                img_src = None
+            results.append({'url': res_url,
+                            'title': title,
+                            'publishedDate': published_date,
+                            'content': content,
+                            'img_src': img_src})
 
-    # return results
     return results
+
+
+# get supported languages from their site
+def _fetch_supported_languages(resp):
+    # list of regions is embedded in page as a js object
+    response_text = resp.text
+    response_text = response_text[response_text.find('regionalisation'):]
+    response_text = response_text[response_text.find('{'):response_text.find(');')]
+
+    regions_json = loads(response_text)
+
+    supported_languages = []
+    for lang in regions_json['languages'].values():
+        if lang['code'] == 'nb':
+            lang['code'] = 'no'
+        for country in lang['countries']:
+            supported_languages.append(lang['code'] + '-' + country)
+
+    return supported_languages

searx/engines/reddit.py

@@ -11,9 +11,8 @@
 """
 
 import json
-from urllib import urlencode
-from urlparse import urlparse, urljoin
 from datetime import datetime
+from searx.url_utils import urlencode, urljoin, urlparse
 
 # engine dependent config
 categories = ['general', 'images', 'news', 'social media']
@@ -26,8 +25,7 @@ search_url = base_url + 'search.json?{query}'
 
 # do search-request
 def request(query, params):
-    query = urlencode({'q': query,
-                       'limit': page_size})
+    query = urlencode({'q': query, 'limit': page_size})
     params['url'] = search_url.format(query=query)
 
     return params

searx/engines/scanr_structures.py

@@ -10,9 +10,7 @@
  @parse       url, title, content, img_src
 """
 
-from urllib import urlencode
 from json import loads, dumps
-from dateutil import parser
 from searx.utils import html_to_text
 
 # engine dependent config
@@ -48,7 +46,7 @@ def response(resp):
     search_res = loads(resp.text)
 
     # return empty array if there are no results
-    if search_res.get('total') < 1:
+    if search_res.get('total', 0) < 1:
         return []
 
     # parse results

searx/engines/searchcode_code.py

@@ -10,8 +10,8 @@
  @parse       url, title, content
 """
 
-from urllib import urlencode
 from json import loads
+from searx.url_utils import urlencode
 
 
 # engine dependent config
@@ -31,8 +31,7 @@ code_endings = {'cs': 'c#',
 
 # do search-request
 def request(query, params):
-    params['url'] = search_url.format(query=urlencode({'q': query}),
-                                      pageno=params['pageno'] - 1)
+    params['url'] = search_url.format(query=urlencode({'q': query}), pageno=params['pageno'] - 1)
 
     return params
 

searx/engines/searchcode_doc.py

@@ -10,8 +10,8 @@
  @parse       url, title, content
 """
 
-from urllib import urlencode
 from json import loads
+from searx.url_utils import urlencode
 
 # engine dependent config
 categories = ['it']
@@ -24,8 +24,7 @@ search_url = url + 'api/search_IV/?{query}&p={pageno}'
 
 # do search-request
 def request(query, params):
-    params['url'] = search_url.format(query=urlencode({'q': query}),
-                                      pageno=params['pageno'] - 1)
+    params['url'] = search_url.format(query=urlencode({'q': query}), pageno=params['pageno'] - 1)
 
     return params
 

searx/engines/seedpeer.py

@@ -8,11 +8,9 @@
 # @stable      yes (HTML can change)
 # @parse       url, title, content, seed, leech, magnetlink
 
-from urlparse import urljoin
-from urllib import quote
 from lxml import html
 from operator import itemgetter
-from searx.engines.xpath import extract_text
+from searx.url_utils import quote, urljoin
 
 
 url = 'http://www.seedpeer.eu/'

searx/engines/soundcloud.py

@@ -11,13 +11,17 @@
 """
 
 import re
-from StringIO import StringIO
 from json import loads
-from lxml import etree
-from urllib import urlencode, quote_plus
+from lxml import html
 from dateutil import parser
 from searx import logger
 from searx.poolrequests import get as http_get
+from searx.url_utils import quote_plus, urlencode
+
+try:
+    from cStringIO import StringIO
+except:
+    from io import StringIO
 
 # engine dependent config
 categories = ['music']
@@ -36,14 +40,15 @@ embedded_url = '<iframe width="100%" height="166" ' +\
     'scrolling="no" frameborder="no" ' +\
     'data-src="https://w.soundcloud.com/player/?url={uri}"></iframe>'
 
+cid_re = re.compile(r'client_id:"([^"]*)"', re.I | re.U)
+
 
 def get_client_id():
     response = http_get("https://soundcloud.com")
-    rx_namespace = {"re": "http://exslt.org/regular-expressions"}
 
     if response.ok:
-        tree = etree.parse(StringIO(response.content), etree.HTMLParser())
-        script_tags = tree.xpath("//script[re:match(@src, '(.*app.*js)')]", namespaces=rx_namespace)
+        tree = html.fromstring(response.content)
+        script_tags = tree.xpath("//script[contains(@src, '/assets/app')]")
         app_js_urls = [script_tag.get('src') for script_tag in script_tags if script_tag is not None]
 
         # extracts valid app_js urls from soundcloud.com content
@@ -51,7 +56,7 @@ def get_client_id():
             # gets app_js and searches for the clientid
             response = http_get(app_js_url)
             if response.ok:
-                cids = re.search(r'client_id:"([^"]*)"', response.content, re.M | re.I)
+                cids = cid_re.search(response.text)
                 if cids is not None and len(cids.groups()):
                     return cids.groups()[0]
     logger.warning("Unable to fetch guest client_id from SoundCloud, check parser!")

searx/engines/spotify.py

@@ -11,7 +11,7 @@
 """
 
 from json import loads
-from urllib import urlencode
+from searx.url_utils import urlencode
 
 # engine dependent config
 categories = ['music']
@@ -29,8 +29,7 @@ embedded_url = '<iframe data-src="https://embed.spotify.com/?uri=spotify:track:{
 def request(query, params):
     offset = (params['pageno'] - 1) * 20
 
-    params['url'] = search_url.format(query=urlencode({'q': query}),
-                                      offset=offset)
+    params['url'] = search_url.format(query=urlencode({'q': query}), offset=offset)
 
     return params
 

searx/engines/stackoverflow.py

@@ -10,10 +10,9 @@
  @parse       url, title, content
 """
 
-from urlparse import urljoin
-from urllib import urlencode
 from lxml import html
 from searx.engines.xpath import extract_text
+from searx.url_utils import urlencode, urljoin
 
 # engine dependent config
 categories = ['it']
@@ -31,8 +30,7 @@ content_xpath = './/div[@class="excerpt"]'
 
 # do search-request
 def request(query, params):
-    params['url'] = search_url.format(query=urlencode({'q': query}),
-                                      pageno=params['pageno'])
+    params['url'] = search_url.format(query=urlencode({'q': query}), pageno=params['pageno'])
 
     return params
 

searx/engines/startpage.py

@@ -56,7 +56,7 @@ def request(query, params):
 def response(resp):
     results = []
 
-    dom = html.fromstring(resp.content)
+    dom = html.fromstring(resp.text)
 
     # parse results
     for result in dom.xpath(results_xpath):

searx/engines/subtitleseeker.py

@@ -10,10 +10,10 @@
  @parse       url, title, content
 """
 
-from urllib import quote_plus
 from lxml import html
 from searx.languages import language_codes
 from searx.engines.xpath import extract_text
+from searx.url_utils import quote_plus
 
 # engine dependent config
 categories = ['videos']

searx/engines/swisscows.py

@@ -11,9 +11,9 @@
 """
 
 from json import loads
-from urllib import urlencode, unquote
 import re
 from lxml.html import fromstring
+from searx.url_utils import unquote, urlencode
 
 # engine dependent config
 categories = ['general', 'images']
@@ -27,10 +27,10 @@ search_string = '?{query}&page={page}'
 supported_languages_url = base_url
 
 # regex
-regex_json = re.compile(r'initialData: {"Request":(.|\n)*},\s*environment')
-regex_json_remove_start = re.compile(r'^initialData:\s*')
-regex_json_remove_end = re.compile(r',\s*environment$')
-regex_img_url_remove_start = re.compile(r'^https?://i\.swisscows\.ch/\?link=')
+regex_json = re.compile(b'initialData: {"Request":(.|\n)*},\s*environment')
+regex_json_remove_start = re.compile(b'^initialData:\s*')
+regex_json_remove_end = re.compile(b',\s*environment$')
+regex_img_url_remove_start = re.compile(b'^https?://i\.swisscows\.ch/\?link=')
 
 
 # do search-request
@@ -45,10 +45,9 @@ def request(query, params):
         ui_language = params['language'].split('-')[0]
 
     search_path = search_string.format(
-        query=urlencode({'query': query,
-                         'uiLanguage': ui_language,
-                         'region': region}),
-        page=params['pageno'])
+        query=urlencode({'query': query, 'uiLanguage': ui_language, 'region': region}),
+        page=params['pageno']
+    )
 
     # image search query is something like 'image?{query}&page={page}'
     if params['category'] == 'images':
@@ -63,14 +62,14 @@ def request(query, params):
 def response(resp):
     results = []
 
-    json_regex = regex_json.search(resp.content)
+    json_regex = regex_json.search(resp.text)
 
     # check if results are returned
     if not json_regex:
         return []
 
-    json_raw = regex_json_remove_end.sub('', regex_json_remove_start.sub('', json_regex.group()))
-    json = loads(json_raw)
+    json_raw = regex_json_remove_end.sub(b'', regex_json_remove_start.sub(b'', json_regex.group()))
+    json = loads(json_raw.decode('utf-8'))
 
     # parse results
     for result in json['Results'].get('items', []):
@@ -78,7 +77,7 @@ def response(resp):
 
         # parse image results
         if result.get('ContentType', '').startswith('image'):
-            img_url = unquote(regex_img_url_remove_start.sub('', result['Url']))
+            img_url = unquote(regex_img_url_remove_start.sub(b'', result['Url'].encode('utf-8')).decode('utf-8'))
 
             # append result
             results.append({'url': result['SourceUrl'],
@@ -100,7 +99,7 @@ def response(resp):
     # parse images
     for result in json.get('Images', []):
         # decode image url
-        img_url = unquote(regex_img_url_remove_start.sub('', result['Url']))
+        img_url = unquote(regex_img_url_remove_start.sub(b'', result['Url'].encode('utf-8')).decode('utf-8'))
 
         # append result
         results.append({'url': result['SourceUrl'],
@@ -120,6 +119,8 @@ def _fetch_supported_languages(resp):
     options = dom.xpath('//div[@id="regions-popup"]//ul/li/a')
     for option in options:
         code = option.xpath('./@data-val')[0]
+        if code.startswith('nb-'):
+            code = code.replace('nb', 'no', 1)
         supported_languages.append(code)
 
     return supported_languages

searx/engines/tokyotoshokan.py

@@ -11,11 +11,11 @@
 """
 
 import re
-from urllib import urlencode
 from lxml import html
 from searx.engines.xpath import extract_text
 from datetime import datetime
 from searx.engines.nyaa import int_or_zero, get_filesize_mul
+from searx.url_utils import urlencode
 
 # engine dependent config
 categories = ['files', 'videos', 'music']
@@ -28,8 +28,7 @@ search_url = base_url + 'search.php?{query}'
 
 # do search-request
 def request(query, params):
-    query = urlencode({'page': params['pageno'],
-                       'terms': query})
+    query = urlencode({'page': params['pageno'], 'terms': query})
     params['url'] = search_url.format(query=query)
     return params
 
@@ -50,7 +49,7 @@ def response(resp):
     size_re = re.compile(r'Size:\s*([\d.]+)(TB|GB|MB|B)', re.IGNORECASE)
 
     # processing the results, two rows at a time
-    for i in xrange(0, len(rows), 2):
+    for i in range(0, len(rows), 2):
         # parse the first row
         name_row = rows[i]
 
@@ -79,14 +78,14 @@ def response(resp):
                     groups = size_re.match(item).groups()
                     multiplier = get_filesize_mul(groups[1])
                     params['filesize'] = int(multiplier * float(groups[0]))
-                except Exception as e:
+                except:
                     pass
             elif item.startswith('Date:'):
                 try:
                     # Date: 2016-02-21 21:44 UTC
                     date = datetime.strptime(item, 'Date: %Y-%m-%d %H:%M UTC')
                     params['publishedDate'] = date
-                except Exception as e:
+                except:
                     pass
             elif item.startswith('Comment:'):
                 params['content'] = item

searx/engines/torrentz.py

@@ -12,11 +12,11 @@
 """
 
 import re
-from urllib import urlencode
 from lxml import html
-from searx.engines.xpath import extract_text
 from datetime import datetime
 from searx.engines.nyaa import int_or_zero, get_filesize_mul
+from searx.engines.xpath import extract_text
+from searx.url_utils import urlencode
 
 # engine dependent config
 categories = ['files', 'videos', 'music']
@@ -70,7 +70,7 @@ def response(resp):
             size_str = result.xpath('./dd/span[@class="s"]/text()')[0]
             size, suffix = size_str.split()
             params['filesize'] = int(size) * get_filesize_mul(suffix)
-        except Exception as e:
+        except:
             pass
 
         # does our link contain a valid SHA1 sum?
@@ -84,7 +84,7 @@ def response(resp):
             # Fri, 25 Mar 2016 16:29:01
             date = datetime.strptime(date_str, '%a, %d %b %Y %H:%M:%S')
             params['publishedDate'] = date
-        except Exception as e:
+        except:
             pass
 
         results.append(params)

searx/engines/translated.py

@@ -9,8 +9,12 @@
  @parse       url, title, content
 """
 import re
+from sys import version_info
 from searx.utils import is_valid_lang
 
+if version_info[0] == 3:
+    unicode = str
+
 categories = ['general']
 url = u'http://api.mymemory.translated.net/get?q={query}&langpair={from_lang}|{to_lang}{key}'
 web_url = u'http://mymemory.translated.net/en/{from_lang}/{to_lang}/{query}'

searx/engines/twitter.py

@@ -12,11 +12,10 @@
  @todo        publishedDate
 """
 
-from urlparse import urljoin
-from urllib import urlencode
 from lxml import html
 from datetime import datetime
 from searx.engines.xpath import extract_text
+from searx.url_utils import urlencode, urljoin
 
 # engine dependent config
 categories = ['social media']
@@ -28,6 +27,7 @@ search_url = base_url + 'search?'
 
 # specific xpath variables
 results_xpath = '//li[@data-item-type="tweet"]'
+avatar_xpath = './/img[contains(@class, "avatar")]/@src'
 link_xpath = './/small[@class="time"]//a'
 title_xpath = './/span[contains(@class, "username")]'
 content_xpath = './/p[contains(@class, "tweet-text")]'
@@ -58,6 +58,8 @@ def response(resp):
         try:
             link = tweet.xpath(link_xpath)[0]
             content = extract_text(tweet.xpath(content_xpath)[0])
+            img_src = tweet.xpath(avatar_xpath)[0]
+            img_src = img_src.replace('_bigger', '_normal')
         except Exception:
             continue
 
@@ -72,12 +74,14 @@ def response(resp):
             results.append({'url': url,
                             'title': title,
                             'content': content,
+                            'img_src': img_src,
                             'publishedDate': publishedDate})
         else:
             # append result
             results.append({'url': url,
                             'title': title,
-                            'content': content})
+                            'content': content,
+                            'img_src': img_src})
 
     # return results
     return results

searx/engines/vimeo.py

@@ -13,8 +13,8 @@
 # @todo        set content-parameter with correct data
 
 from json import loads
-from urllib import urlencode
 from dateutil import parser
+from searx.url_utils import urlencode
 
 # engine dependent config
 categories = ['videos']

searx/engines/wikidata.py

@@ -14,12 +14,11 @@
 from searx import logger
 from searx.poolrequests import get
 from searx.engines.xpath import extract_text
-from searx.utils import format_date_by_locale
 from searx.engines.wikipedia import _fetch_supported_languages, supported_languages_url
+from searx.url_utils import urlencode
 
 from json import loads
 from lxml.html import fromstring
-from urllib import urlencode
 
 logger = logger.getChild('wikidata')
 result_count = 1
@@ -62,14 +61,13 @@ def request(query, params):
         language = 'en'
 
     params['url'] = url_search.format(
-        query=urlencode({'label': query,
-                        'language': language}))
+        query=urlencode({'label': query, 'language': language}))
     return params
 
 
 def response(resp):
     results = []
-    html = fromstring(resp.content)
+    html = fromstring(resp.text)
     wikidata_ids = html.xpath(wikidata_ids_xpath)
 
     language = resp.search_params['language'].split('-')[0]
@@ -78,10 +76,9 @@ def response(resp):
 
     # TODO: make requests asynchronous to avoid timeout when result_count > 1
     for wikidata_id in wikidata_ids[:result_count]:
-        url = url_detail.format(query=urlencode({'page': wikidata_id,
-                                                'uselang': language}))
+        url = url_detail.format(query=urlencode({'page': wikidata_id, 'uselang': language}))
         htmlresponse = get(url)
-        jsonresponse = loads(htmlresponse.content)
+        jsonresponse = loads(htmlresponse.text)
         results += getDetail(jsonresponse, wikidata_id, language, resp.search_params['language'])
 
     return results

searx/engines/wikipedia.py

@@ -11,13 +11,12 @@
 """
 
 from json import loads
-from urllib import urlencode, quote
 from lxml.html import fromstring
-
+from searx.url_utils import quote, urlencode
 
 # search-url
-base_url = 'https://{language}.wikipedia.org/'
-search_postfix = 'w/api.php?'\
+base_url = u'https://{language}.wikipedia.org/'
+search_url = base_url + u'w/api.php?'\
     'action=query'\
     '&format=json'\
     '&{query}'\
@@ -37,16 +36,16 @@ def url_lang(lang):
     else:
         language = lang
 
-    return base_url.format(language=language)
+    return language
 
 
 # do search-request
 def request(query, params):
     if query.islower():
-        query += '|' + query.title()
+        query = u'{0}|{1}'.format(query.decode('utf-8'), query.decode('utf-8').title()).encode('utf-8')
 
-    params['url'] = url_lang(params['language']) \
-        + search_postfix.format(query=urlencode({'titles': query}))
+    params['url'] = search_url.format(query=urlencode({'titles': query}),
+                                      language=url_lang(params['language']))
 
     return params
 
@@ -78,7 +77,7 @@ def extract_first_paragraph(content, title, image):
 def response(resp):
     results = []
 
-    search_result = loads(resp.content)
+    search_result = loads(resp.text)
 
     # wikipedia article's unique id
     # first valid id is assumed to be the requested article
@@ -99,11 +98,9 @@ def response(resp):
     extract = page.get('extract')
 
     summary = extract_first_paragraph(extract, title, image)
-    if not summary:
-        return []
 
     # link to wikipedia article
-    wikipedia_link = url_lang(resp.search_params['language']) \
+    wikipedia_link = base_url.format(language=url_lang(resp.search_params['language'])) \
         + 'wiki/' + quote(title.replace(' ', '_').encode('utf8'))
 
     results.append({'url': wikipedia_link, 'title': title})

searx/engines/wolframalpha_api.py

@@ -8,8 +8,8 @@
 # @stable      yes
 # @parse       url, infobox
 
-from urllib import urlencode
 from lxml import etree
+from searx.url_utils import urlencode
 
 # search-url
 search_url = 'https://api.wolframalpha.com/v2/query?appid={api_key}&{query}'
@@ -37,8 +37,7 @@ image_pods = {'VisualRepresentation',
 
 # do search-request
 def request(query, params):
-    params['url'] = search_url.format(query=urlencode({'input': query}),
-                                      api_key=api_key)
+    params['url'] = search_url.format(query=urlencode({'input': query}), api_key=api_key)
     params['headers']['Referer'] = site_url.format(query=urlencode({'i': query}))
 
     return params
@@ -56,7 +55,7 @@ def replace_pua_chars(text):
                  u'\uf74e': 'i',        # imaginary number
                  u'\uf7d9': '='}        # equals sign
 
-    for k, v in pua_chars.iteritems():
+    for k, v in pua_chars.items():
         text = text.replace(k, v)
 
     return text
@@ -66,7 +65,7 @@ def replace_pua_chars(text):
 def response(resp):
     results = []
 
-    search_results = etree.XML(resp.content)
+    search_results = etree.XML(resp.text)
 
     # return empty array if there are no results
     if search_results.xpath(failure_xpath):
@@ -120,10 +119,10 @@ def response(resp):
     # append infobox
     results.append({'infobox': infobox_title,
                     'attributes': result_chunks,
-                    'urls': [{'title': 'Wolfram|Alpha', 'url': resp.request.headers['Referer'].decode('utf8')}]})
+                    'urls': [{'title': 'Wolfram|Alpha', 'url': resp.request.headers['Referer']}]})
 
     # append link to site
-    results.append({'url': resp.request.headers['Referer'].decode('utf8'),
+    results.append({'url': resp.request.headers['Referer'],
                     'title': title,
                     'content': result_content})
 

searx/engines/wolframalpha_noapi.py

@@ -10,10 +10,9 @@
 
 from json import loads
 from time import time
-from urllib import urlencode
-from lxml.etree import XML
 
 from searx.poolrequests import get as http_get
+from searx.url_utils import urlencode
 
 # search-url
 url = 'https://www.wolframalpha.com/'
@@ -62,7 +61,7 @@ obtain_token()
 # do search-request
 def request(query, params):
     # obtain token if last update was more than an hour
-    if time() - token['last_updated'] > 3600:
+    if time() - (token['last_updated'] or 0) > 3600:
         obtain_token()
     params['url'] = search_url.format(query=urlencode({'input': query}), token=token['value'])
     params['headers']['Referer'] = referer_url.format(query=urlencode({'i': query}))
@@ -112,9 +111,9 @@ def response(resp):
 
     results.append({'infobox': infobox_title,
                     'attributes': result_chunks,
-                    'urls': [{'title': 'Wolfram|Alpha', 'url': resp.request.headers['Referer'].decode('utf8')}]})
+                    'urls': [{'title': 'Wolfram|Alpha', 'url': resp.request.headers['Referer']}]})
 
-    results.append({'url': resp.request.headers['Referer'].decode('utf8'),
+    results.append({'url': resp.request.headers['Referer'],
                     'title': 'Wolfram|Alpha (' + infobox_title + ')',
                     'content': result_content})
 

searx/engines/www1x.py

@@ -10,11 +10,9 @@
  @parse       url, title, thumbnail, img_src, content
 """
 
-from urllib import urlencode
-from urlparse import urljoin
 from lxml import html
-import string
 import re
+from searx.url_utils import urlencode, urljoin
 
 # engine dependent config
 categories = ['images']
@@ -55,7 +53,7 @@ def response(resp):
         cur_element += result_part
 
         # fix xml-error
-        cur_element = string.replace(cur_element, '"></a>', '"/></a>')
+        cur_element = cur_element.replace('"></a>', '"/></a>')
 
         dom = html.fromstring(cur_element)
         link = dom.xpath('//a')[0]

searx/engines/www500px.py

@@ -13,8 +13,7 @@
 """
 
 from json import loads
-from urllib import urlencode
-from urlparse import urljoin
+from searx.url_utils import urlencode, urljoin
 
 # engine dependent config
 categories = ['images']

searx/engines/xpath.py

@@ -1,13 +1,13 @@
 from lxml import html
-from urllib import urlencode, unquote
-from urlparse import urlparse, urljoin
 from lxml.etree import _ElementStringResult, _ElementUnicodeResult
 from searx.utils import html_to_text
+from searx.url_utils import unquote, urlencode, urljoin, urlparse
 
 search_url = None
 url_xpath = None
 content_xpath = None
 title_xpath = None
+paging = False
 suggestion_xpath = ''
 results_xpath = ''
 
@@ -31,8 +31,6 @@ if xpath_results is a string element, then it's already done
 def extract_text(xpath_results):
     if type(xpath_results) == list:
         # it's list of result : concat everything using recursive call
-        if not xpath_results:
-            raise Exception('Empty url resultset')
         result = ''
         for e in xpath_results:
             result = result + extract_text(e)
@@ -48,12 +46,14 @@ def extract_text(xpath_results):
 
 
 def extract_url(xpath_results, search_url):
+    if xpath_results == []:
+        raise Exception('Empty url resultset')
     url = extract_text(xpath_results)
 
     if url.startswith('//'):
         # add http or https to this kind of url //example.com/
         parsed_search_url = urlparse(search_url)
-        url = parsed_search_url.scheme + url
+        url = u'{0}:{1}'.format(parsed_search_url.scheme, url)
     elif url.startswith('/'):
         # fix relative url to the search engine
         url = urljoin(search_url, url)
@@ -103,8 +103,8 @@ def response(resp):
     if results_xpath:
         for result in dom.xpath(results_xpath):
             url = extract_url(result.xpath(url_xpath), search_url)
-            title = extract_text(result.xpath(title_xpath)[0])
-            content = extract_text(result.xpath(content_xpath)[0])
+            title = extract_text(result.xpath(title_xpath))
+            content = extract_text(result.xpath(content_xpath))
             results.append({'url': url, 'title': title, 'content': content})
     else:
         for url, title, content in zip(

searx/engines/yacy.py

@@ -13,8 +13,8 @@
 # @todo        parse video, audio and file results
 
 from json import loads
-from urllib import urlencode
 from dateutil import parser
+from searx.url_utils import urlencode
 
 from searx.utils import html_to_text
 

searx/engines/yahoo.py

@@ -11,10 +11,9 @@
  @parse       url, title, content, suggestion
 """
 
-from urllib import urlencode
-from urlparse import unquote
 from lxml import html
 from searx.engines.xpath import extract_text, extract_url
+from searx.url_utils import unquote, urlencode
 
 # engine dependent config
 categories = ['general']

searx/engines/yahoo_news.py

@@ -9,13 +9,13 @@
 # @stable      no (HTML can change)
 # @parse       url, title, content, publishedDate
 
-from urllib import urlencode
+import re
+from datetime import datetime, timedelta
 from lxml import html
 from searx.engines.xpath import extract_text, extract_url
 from searx.engines.yahoo import parse_url, _fetch_supported_languages, supported_languages_url
-from datetime import datetime, timedelta
-import re
 from dateutil import parser
+from searx.url_utils import urlencode
 
 # engine dependent config
 categories = ['news']
@@ -80,16 +80,19 @@ def response(resp):
 
         # still useful ?
         if re.match("^[0-9]+ minute(s|) ago$", publishedDate):
-            publishedDate = datetime.now() - timedelta(minutes=int(re.match(r'\d+', publishedDate).group()))  # noqa
+            publishedDate = datetime.now() - timedelta(minutes=int(re.match(r'\d+', publishedDate).group()))
+        elif re.match("^[0-9]+ days? ago$", publishedDate):
+            publishedDate = datetime.now() - timedelta(days=int(re.match(r'\d+', publishedDate).group()))
+        elif re.match("^[0-9]+ hour(s|), [0-9]+ minute(s|) ago$", publishedDate):
+            timeNumbers = re.findall(r'\d+', publishedDate)
+            publishedDate = datetime.now()\
+                - timedelta(hours=int(timeNumbers[0]))\
+                - timedelta(minutes=int(timeNumbers[1]))
         else:
-            if re.match("^[0-9]+ hour(s|), [0-9]+ minute(s|) ago$",
-                        publishedDate):
-                timeNumbers = re.findall(r'\d+', publishedDate)
-                publishedDate = datetime.now()\
-                    - timedelta(hours=int(timeNumbers[0]))\
-                    - timedelta(minutes=int(timeNumbers[1]))
-            else:
+            try:
                 publishedDate = parser.parse(publishedDate)
+            except:
+                publishedDate = datetime.now()
 
         if publishedDate.year == 1900:
             publishedDate = publishedDate.replace(year=datetime.now().year)

searx/engines/yandex.py

@@ -9,9 +9,9 @@
  @parse       url, title, content
 """
 
-from urllib import urlencode
 from lxml import html
-from searx.search import logger
+from searx import logger
+from searx.url_utils import urlencode
 
 logger = logger.getChild('yandex engine')
 

searx/engines/youtube_api.py

@@ -9,8 +9,8 @@
 # @parse       url, title, content, publishedDate, thumbnail, embedded
 
 from json import loads
-from urllib import urlencode
 from dateutil import parser
+from searx.url_utils import urlencode
 
 # engine dependent config
 categories = ['videos', 'music']

searx/engines/youtube_noapi.py

@@ -8,10 +8,10 @@
 # @stable      no
 # @parse       url, title, content, publishedDate, thumbnail, embedded
 
-from urllib import quote_plus
 from lxml import html
 from searx.engines.xpath import extract_text
 from searx.utils import list_get
+from searx.url_utils import quote_plus
 
 # engine dependent config
 categories = ['videos', 'music']

searx/exceptions.py

@@ -0,0 +1,32 @@
+'''
+searx is free software: you can redistribute it and/or modify
+it under the terms of the GNU Affero General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+searx is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU Affero General Public License for more details.
+
+You should have received a copy of the GNU Affero General Public License
+along with searx. If not, see < http://www.gnu.org/licenses/ >.
+
+(C) 2017- by Alexandre Flament, <alex@al-f.net>
+'''
+
+
+class SearxException(Exception):
+    pass
+
+
+class SearxParameterException(SearxException):
+
+    def __init__(self, name, value):
+        if value == '' or value is None:
+            message = 'Empty ' + name + ' parameter'
+        else:
+            message = 'Invalid value "' + value + '" for parameter ' + name
+        super(SearxParameterException, self).__init__(message)
+        self.parameter_name = name
+        self.parameter_value = value

searx/languages.py

@@ -5,9 +5,6 @@
 language_codes = (
     (u"ar-SA", u"العربية", u"", u"Arabic"),
     (u"bg-BG", u"Български", u"", u"Bulgarian"),
-    (u"ca", u"Català", u"", u"Catalan"),
-    (u"ca-CT", u"Català", u"", u"Catalan"),
-    (u"ca-ES", u"Català", u"Espanya", u"Catalan"),
     (u"cs-CZ", u"Čeština", u"", u"Czech"),
     (u"da-DK", u"Dansk", u"", u"Danish"),
     (u"de", u"Deutsch", u"", u"German"),
@@ -18,7 +15,9 @@ language_codes = (
     (u"en", u"English", u"", u"English"),
     (u"en-AU", u"English", u"Australia", u"English"),
     (u"en-CA", u"English", u"Canada", u"English"),
+    (u"en-CY", u"English", u"Cyprus", u"English"),
     (u"en-GB", u"English", u"United Kingdom", u"English"),
+    (u"en-GD", u"English", u"Grenada", u"English"),
     (u"en-ID", u"English", u"Indonesia", u"English"),
     (u"en-IE", u"English", u"Ireland", u"English"),
     (u"en-IN", u"English", u"India", u"English"),
@@ -54,6 +53,7 @@ language_codes = (
     (u"ko-KR", u"한국어", u"", u"Korean"),
     (u"lt-LT", u"Lietuvių", u"", u"Lithuanian"),
     (u"lv-LV", u"Latviešu", u"", u"Latvian"),
+    (u"ms-MY", u"Bahasa Melayu", u"", u"Malay"),
     (u"nl", u"Nederlands", u"", u"Dutch"),
     (u"nl-BE", u"Nederlands", u"België", u"Dutch"),
     (u"nl-NL", u"Nederlands", u"Nederland", u"Dutch"),
@@ -69,7 +69,6 @@ language_codes = (
     (u"sv-SE", u"Svenska", u"", u"Swedish"),
     (u"th-TH", u"ไทย", u"", u"Thai"),
     (u"tr-TR", u"Türkçe", u"", u"Turkish"),
-    (u"uk-UA", u"Українська", u"", u"Ukrainian"),
     (u"vi-VN", u"Tiếng Việt", u"", u"Vietnamese"),
     (u"zh", u"中文", u"", u"Chinese"),
     (u"zh-CN", u"中文", u"中国", u"Chinese"),

searx/plugins/__init__.py

@@ -14,9 +14,12 @@ along with searx. If not, see < http://www.gnu.org/licenses/ >.
 
 (C) 2015 by Adam Tauber, <asciimoo@gmail.com>
 '''
-from sys import exit
+from sys import exit, version_info
 from searx import logger
 
+if version_info[0] == 3:
+    unicode = str
+
 logger = logger.getChild('plugins')
 
 from searx.plugins import (doai_rewrite,
@@ -63,9 +66,9 @@ class PluginStore():
             plugin.id = plugin.name.replace(' ', '_')
             self.plugins.append(plugin)
 
-    def call(self, plugin_type, request, *args, **kwargs):
+    def call(self, ordered_plugin_list, plugin_type, request, *args, **kwargs):
         ret = True
-        for plugin in request.user_plugins:
+        for plugin in ordered_plugin_list:
             if hasattr(plugin, plugin_type):
                 ret = getattr(plugin, plugin_type)(request, *args, **kwargs)
                 if not ret:

searx/plugins/doai_rewrite.py

@@ -1,6 +1,6 @@
 from flask_babel import gettext
 import re
-from urlparse import urlparse, parse_qsl
+from searx.url_utils import urlparse, parse_qsl
 
 regex = re.compile(r'10\.\d{4,9}/[^\s]+')
 

searx/plugins/https_rewrite.py

@@ -16,14 +16,17 @@ along with searx. If not, see < http://www.gnu.org/licenses/ >.
 '''
 
 import re
-from urlparse import urlparse
+import sys
 from lxml import etree
 from os import listdir, environ
 from os.path import isfile, isdir, join
 from searx.plugins import logger
 from flask_babel import gettext
 from searx import searx_dir
+from searx.url_utils import urlparse
 
+if sys.version_info[0] == 3:
+    unicode = str
 
 name = "HTTPS rewrite"
 description = gettext('Rewrite HTTP links to HTTPS if possible')

searx/plugins/self_info.py

@@ -22,7 +22,7 @@ default_on = True
 
 
 # Self User Agent regex
-p = re.compile('.*user[ -]agent.*', re.IGNORECASE)
+p = re.compile(b'.*user[ -]agent.*', re.IGNORECASE)
 
 
 # attach callback to the post search hook
@@ -31,7 +31,7 @@ p = re.compile('.*user[ -]agent.*', re.IGNORECASE)
 def post_search(request, search):
     if search.search_query.pageno > 1:
         return True
-    if search.search_query.query == 'ip':
+    if search.search_query.query == b'ip':
         x_forwarded_for = request.headers.getlist("X-Forwarded-For")
         if x_forwarded_for:
             ip = x_forwarded_for[0]

searx/plugins/tracker_url_remover.py

@@ -17,7 +17,7 @@ along with searx. If not, see < http://www.gnu.org/licenses/ >.
 
 from flask_babel import gettext
 import re
-from urlparse import urlunparse
+from searx.url_utils import urlunparse
 
 regexes = {re.compile(r'utm_[^&]+&?'),
            re.compile(r'(wkey|wemail)[^&]+&?'),

searx/preferences.py

@@ -23,7 +23,7 @@ class Setting(object):
     def __init__(self, default_value, **kwargs):
         super(Setting, self).__init__()
         self.value = default_value
-        for key, value in kwargs.iteritems():
+        for key, value in kwargs.items():
             setattr(self, key, value)
 
         self._post_init()
@@ -38,7 +38,7 @@ class Setting(object):
         return self.value
 
     def save(self, name, resp):
-        resp.set_cookie(name, bytes(self.value), max_age=COOKIE_MAX_AGE)
+        resp.set_cookie(name, self.value, max_age=COOKIE_MAX_AGE)
 
 
 class StringSetting(Setting):
@@ -107,6 +107,8 @@ class SearchLanguageSetting(EnumStringSetting):
                 pass
             elif lang in self.choices:
                 data = lang
+            elif data == 'nb-NO':
+                data = 'no-NO'
             elif data == 'ar-XA':
                 data = 'ar-SA'
             else:
@@ -130,7 +132,8 @@ class MapSetting(Setting):
         self.key = data
 
     def save(self, name, resp):
-        resp.set_cookie(name, bytes(self.key), max_age=COOKIE_MAX_AGE)
+        if hasattr(self, 'key'):
+            resp.set_cookie(name, self.key, max_age=COOKIE_MAX_AGE)
 
 
 class SwitchableSetting(Setting):
@@ -191,7 +194,7 @@ class EnginesSetting(SwitchableSetting):
     def _post_init(self):
         super(EnginesSetting, self)._post_init()
         transformed_choices = []
-        for engine_name, engine in self.choices.iteritems():
+        for engine_name, engine in self.choices.items():
             for category in engine.categories:
                 transformed_choice = dict()
                 transformed_choice['default_on'] = not engine.disabled
@@ -238,9 +241,9 @@ class Preferences(object):
                                    'language': SearchLanguageSetting(settings['search']['language'],
                                                                      choices=LANGUAGE_CODES),
                                    'locale': EnumStringSetting(settings['ui']['default_locale'],
-                                                               choices=settings['locales'].keys() + ['']),
+                                                               choices=list(settings['locales'].keys()) + ['']),
                                    'autocomplete': EnumStringSetting(settings['search']['autocomplete'],
-                                                                     choices=autocomplete.backends.keys() + ['']),
+                                                                     choices=list(autocomplete.backends.keys()) + ['']),
                                    'image_proxy': MapSetting(settings['server']['image_proxy'],
                                                              map={'': settings['server']['image_proxy'],
                                                                   '0': False,
@@ -257,7 +260,7 @@ class Preferences(object):
         self.unknown_params = {}
 
     def parse_cookies(self, input_data):
-        for user_setting_name, user_setting in input_data.iteritems():
+        for user_setting_name, user_setting in input_data.items():
             if user_setting_name in self.key_value_settings:
                 self.key_value_settings[user_setting_name].parse(user_setting)
             elif user_setting_name == 'disabled_engines':
@@ -271,7 +274,7 @@ class Preferences(object):
         disabled_engines = []
         enabled_categories = []
         disabled_plugins = []
-        for user_setting_name, user_setting in input_data.iteritems():
+        for user_setting_name, user_setting in input_data.items():
             if user_setting_name in self.key_value_settings:
                 self.key_value_settings[user_setting_name].parse(user_setting)
             elif user_setting_name.startswith('engine_'):
@@ -292,7 +295,7 @@ class Preferences(object):
             return self.key_value_settings[user_setting_name].get_value()
 
     def save(self, resp):
-        for user_setting_name, user_setting in self.key_value_settings.iteritems():
+        for user_setting_name, user_setting in self.key_value_settings.items():
             user_setting.save(user_setting_name, resp)
         self.engines.save(resp)
         self.plugins.save(resp)

searx/query.py

@@ -21,10 +21,14 @@ from searx.languages import language_codes
 from searx.engines import (
     categories, engines, engine_shortcuts
 )
-import string
 import re
+import string
+import sys
 
-VALID_LANGUAGE_CODE = re.compile(r'^[a-z]{2,3}(\-[A-Z]{2})?$')
+if sys.version_info[0] == 3:
+    unicode = str
+
+VALID_LANGUAGE_CODE = re.compile(r'^[a-z]{2,3}(-[a-zA-Z]{2})?$')
 
 
 class RawTextQuery(object):
@@ -68,7 +72,7 @@ class RawTextQuery(object):
 
             # this force a language
             if query_part[0] == ':':
-                lang = query_part[1:].lower()
+                lang = query_part[1:].lower().replace('_', '-')
 
                 # user may set a valid, yet not selectable language
                 if VALID_LANGUAGE_CODE.match(lang):
@@ -86,7 +90,7 @@ class RawTextQuery(object):
                        or lang_id.startswith(lang)\
                        or lang == lang_name\
                        or lang == english_name\
-                       or lang.replace('_', ' ') == country:
+                       or lang.replace('-', ' ') == country:
                         parse_next = True
                         self.languages.append(lang_id)
                         # to ensure best match (first match is not necessarily the best one)
@@ -95,7 +99,7 @@ class RawTextQuery(object):
 
             # this force a engine or category
             if query_part[0] == '!' or query_part[0] == '?':
-                prefix = query_part[1:].replace('-', ' ')
+                prefix = query_part[1:].replace('-', ' ').replace('_', ' ')
 
                 # check if prefix is equal with engine shortcut
                 if prefix in engine_shortcuts:
@@ -146,7 +150,7 @@ class SearchQuery(object):
     """container for all the search parameters (query, language, etc...)"""
 
     def __init__(self, query, engines, categories, lang, safesearch, pageno, time_range):
-        self.query = query
+        self.query = query.encode('utf-8')
         self.engines = engines
         self.categories = categories
         self.lang = lang

searx/results.py

@@ -1,9 +1,13 @@
 import re
+import sys
 from collections import defaultdict
 from operator import itemgetter
 from threading import RLock
-from urlparse import urlparse, unquote
 from searx.engines import engines
+from searx.url_utils import urlparse, unquote
+
+if sys.version_info[0] == 3:
+    basestring = str
 
 CONTENT_LEN_IGNORED_CHARS_REGEX = re.compile(r'[,;:!?\./\\\\ ()-_]', re.M | re.U)
 WHITESPACE_REGEX = re.compile('( |\t|\n)+', re.M | re.U)
@@ -127,6 +131,7 @@ class ResultContainer(object):
         self.infoboxes = []
         self.suggestions = set()
         self.answers = set()
+        self.corrections = set()
         self._number_of_results = []
         self._ordered = False
         self.paging = False
@@ -140,6 +145,9 @@ class ResultContainer(object):
             elif 'answer' in result:
                 self.answers.add(result['answer'])
                 results.remove(result)
+            elif 'correction' in result:
+                self.corrections.add(result['correction'])
+                results.remove(result)
             elif 'infobox' in result:
                 self._merge_infobox(result)
                 results.remove(result)
@@ -242,9 +250,12 @@ class ResultContainer(object):
 
         for i, res in enumerate(results):
             # FIXME : handle more than one category per engine
-            category = engines[res['engine']].categories[0] + ':' + ''\
-                if 'template' not in res\
-                else res['template']
+            res['category'] = engines[res['engine']].categories[0]
+
+            # FIXME : handle more than one category per engine
+            category = engines[res['engine']].categories[0]\
+                + ':' + res.get('template', '')\
+                + ':' + ('img_src' if 'img_src' in res or 'thumbnail' in res else '')
 
             current = None if category not in categoryPositions\
                 else categoryPositions[category]

searx/search.py

@@ -16,8 +16,8 @@ along with searx. If not, see < http://www.gnu.org/licenses/ >.
 '''
 
 import gc
+import sys
 import threading
-from thread import start_new_thread
 from time import time
 from uuid import uuid4
 import requests.exceptions
@@ -27,10 +27,19 @@ from searx.engines import (
 )
 from searx.answerers import ask
 from searx.utils import gen_useragent
-from searx.query import RawTextQuery, SearchQuery
+from searx.query import RawTextQuery, SearchQuery, VALID_LANGUAGE_CODE
 from searx.results import ResultContainer
 from searx import logger
 from searx.plugins import plugins
+from searx.exceptions import SearxParameterException
+
+try:
+    from thread import start_new_thread
+except:
+    from _thread import start_new_thread
+
+if sys.version_info[0] == 3:
+    unicode = str
 
 logger = logger.getChild('search')
 
@@ -182,33 +191,13 @@ def default_request_params():
 
 
 def get_search_query_from_webapp(preferences, form):
-    query = None
-    query_engines = []
-    query_categories = []
-    query_pageno = 1
-    query_lang = 'all'
-    query_time_range = None
+    # no text for the query ?
+    if not form.get('q'):
+        raise SearxParameterException('q', '')
 
     # set blocked engines
     disabled_engines = preferences.engines.get_disabled()
 
-    # set specific language if set
-    query_lang = preferences.get_value('language')
-
-    # safesearch
-    query_safesearch = preferences.get_value('safesearch')
-
-    # TODO better exceptions
-    if not form.get('q'):
-        raise Exception('noquery')
-
-    # set pagenumber
-    pageno_param = form.get('pageno', '1')
-    if not pageno_param.isdigit() or int(pageno_param) < 1:
-        pageno_param = 1
-
-    query_pageno = int(pageno_param)
-
     # parse query, if tags are set, which change
     # the serch engine or search-language
     raw_text_query = RawTextQuery(form['q'], disabled_engines)
@@ -217,6 +206,13 @@ def get_search_query_from_webapp(preferences, form):
     # set query
     query = raw_text_query.getSearchQuery()
 
+    # get and check page number
+    pageno_param = form.get('pageno', '1')
+    if not pageno_param.isdigit() or int(pageno_param) < 1:
+        raise SearxParameterException('pageno', pageno_param)
+    query_pageno = int(pageno_param)
+
+    # get language
     # set specific language if set on request, query or preferences
     # TODO support search with multible languages
     if len(raw_text_query.languages):
@@ -226,10 +222,37 @@ def get_search_query_from_webapp(preferences, form):
     else:
         query_lang = preferences.get_value('language')
 
+    # check language
+    if not VALID_LANGUAGE_CODE.match(query_lang):
+        raise SearxParameterException('language', query_lang)
+
+    # get safesearch
+    if 'safesearch' in form:
+        query_safesearch = form.get('safesearch')
+        # first check safesearch
+        if not query_safesearch.isdigit():
+            raise SearxParameterException('safesearch', query_safesearch)
+        query_safesearch = int(query_safesearch)
+    else:
+        query_safesearch = preferences.get_value('safesearch')
+
+    # safesearch : second check
+    if query_safesearch < 0 or query_safesearch > 2:
+        raise SearxParameterException('safesearch', query_safesearch)
+
+    # get time_range
     query_time_range = form.get('time_range')
 
+    # check time_range
+    if query_time_range not in ('None', None, '', 'day', 'week', 'month', 'year'):
+        raise SearxParameterException('time_range', query_time_range)
+
+    # query_engines
     query_engines = raw_text_query.engines
 
+    # query_categories
+    query_categories = []
+
     # if engines are calculated from query,
     # set categories by using that informations
     if query_engines and raw_text_query.specific:
@@ -352,11 +375,6 @@ class Search(object):
             if search_query.pageno > 1 and not engine.paging:
                 continue
 
-            # if search-language is set and engine does not
-            # provide language-support, skip
-            if search_query.lang != 'all' and not engine.language_support:
-                continue
-
             # if time_range is not supported, skip
             if search_query.time_range and not engine.time_range_support:
                 continue
@@ -377,7 +395,7 @@ class Search(object):
             request_params['time_range'] = search_query.time_range
 
             # append request to list
-            requests.append((selected_engine['name'], search_query.query.encode('utf-8'), request_params))
+            requests.append((selected_engine['name'], search_query.query, request_params))
 
             # update timeout_limit
             timeout_limit = max(timeout_limit, engine.timeout)
@@ -395,19 +413,20 @@ class SearchWithPlugins(Search):
 
     """Similar to the Search class but call the plugins."""
 
-    def __init__(self, search_query, request):
+    def __init__(self, search_query, ordered_plugin_list, request):
         super(SearchWithPlugins, self).__init__(search_query)
+        self.ordered_plugin_list = ordered_plugin_list
         self.request = request
 
     def search(self):
-        if plugins.call('pre_search', self.request, self):
+        if plugins.call(self.ordered_plugin_list, 'pre_search', self.request, self):
             super(SearchWithPlugins, self).search()
 
-        plugins.call('post_search', self.request, self)
+        plugins.call(self.ordered_plugin_list, 'post_search', self.request, self)
 
         results = self.result_container.get_ordered_results()
 
         for result in results:
-            plugins.call('on_result', self.request, self, result)
+            plugins.call(self.ordered_plugin_list, 'on_result', self.request, self, result)
 
         return self.result_container

searx/settings.yml

@@ -16,7 +16,8 @@ server:
     http_protocol_version : "1.0"  # 1.0 and 1.1 are supported
 
 ui:
-    themes_path : "" # Custom ui themes path - leave it blank if you didn't change
+    static_path : "" # Custom static path - leave it blank if you didn't change
+    templates_path : "" # Custom templates path - leave it blank if you didn't change
     default_theme : oscar # ui theme
     default_locale : "" # Default interface locale - leave blank to detect from browser information or use codes from the 'locales' config section
 
@@ -166,6 +167,12 @@ engines:
     shortcut : ddg
     disabled : True
 
+  - name : duckduckgo images
+    engine : duckduckgo_images
+    shortcut : ddi
+    timeout: 3.0
+    disabled : True
+
   - name : etymonline
     engine : xpath
     paging : True
@@ -208,6 +215,18 @@ engines:
 # Or you can use the html non-stable engine, activated by default
     engine : flickr_noapi
 
+  - name : free software directory
+    engine : mediawiki
+    shortcut : fsd
+    categories : it
+    base_url : https://directory.fsf.org/
+    number_of_results : 5
+# what part of a page matches the query string: title, text, nearmatch
+# title - query matches title, text - query matches the text of page, nearmatch - nearmatch in title
+    search_type : title
+    timeout : 5.0
+    disabled : True
+
   - name : frinkiac
     engine : frinkiac
     shortcut : frk
@@ -339,6 +358,17 @@ engines:
     timeout : 4.0
     disabled : True
 
+  - name : library genesis
+    engine : xpath
+    search_url : http://libgen.io/search.php?req={query}
+    url_xpath : //a[contains(@href,"bookfi.net")]/@href
+    title_xpath : //a[contains(@href,"book/")]/text()[1]
+    content_xpath : //td/a[1][contains(@href,"=author")]/text()
+    categories : general
+    timeout : 7.0
+    disabled : True
+    shortcut : lg
+
   - name : lobste.rs
     engine : xpath
     search_url : https://lobste.rs/search?utf8=%E2%9C%93&q={query}&what=stories&order=relevance
@@ -454,6 +484,11 @@ engines:
     shortcut : scc
     disabled : True
 
+  - name : framalibre
+    engine : framalibre
+    shortcut : frl
+    disabled : True
+
 #  - name : searx
 #    engine : searx_engine
 #    shortcut : se
@@ -462,6 +497,17 @@ engines:
 #        - ...
 #    disabled : True
 
+  - name : semantic scholar
+    engine : xpath
+    paging : True
+    search_url : https://www.semanticscholar.org/search?q={query}&sort=relevance&page={pageno}&ae=false
+    results_xpath : //article
+    url_xpath : .//div[@class="search-result-title"]/a/@href
+    title_xpath : .//div[@class="search-result-title"]/a
+    content_xpath : .//div[@class="search-result-abstract"]
+    shortcut : se
+    categories : science
+
   - name : spotify
     engine : spotify
     shortcut : stf
@@ -588,6 +634,10 @@ engines:
     timeout : 10.0
     disabled : True
 
+  - name : 1337x
+    engine : 1337x
+    shortcut : 1337x
+    disabled : True
 
 #The blekko technology and team have joined IBM Watson! -> https://blekko.com/
 #  - name : blekko images
@@ -614,9 +664,11 @@ locales:
     bg : Български (Bulgarian)
     cs : Čeština (Czech)
     de : Deutsch (German)
+    de_DE : Deutsch (German_Germany)
     el_GR : Ελληνικά (Greek_Greece)
     eo : Esperanto (Esperanto)
     es : Español (Spanish)
+    fi : Suomi (Finnish)
     fr : Français (French)
     he : עברית (Hebrew)
     hu : Magyar (Hungarian)
@@ -627,6 +679,8 @@ locales:
     pt_BR : Português (Portuguese_Brazil)
     ro : Română (Romanian)
     ru : Русский (Russian)
+    sk : Slovenčina (Slovak)
     sv : Svenska (Swedish)
     tr : Türkçe (Turkish)
+    uk : українська мова (Ukrainian)
     zh : 中文 (Chinese)

searx/settings_robot.yml

@@ -16,8 +16,9 @@ server:
     http_protocol_version : "1.0"
 
 ui:
-    themes_path : ""
-    default_theme : legacy
+    static_path : ""
+    templates_path : ""
+    default_theme : oscar
     default_locale : ""
 
 outgoing: