[enh] version 0.12.0

[fix] fix duckduckgo's offset

.travis.yml

@@ -4,14 +4,20 @@ cache:
   - npm
   - directories:
     - $HOME/.cache/pip
+addons:
+  firefox: "latest"
 language: python
 python:
   - "2.7"
+  - "3.6"
 before_install:
   - "export DISPLAY=:99.0"
   - "sh -e /etc/init.d/xvfb start"
   - npm install less grunt-cli
   - ( cd searx/static/themes/oscar;npm install; cd - )
+  - mkdir -p ~/drivers; export PATH=~/drivers:$PATH;
+  - GECKODRIVER_URL="https://github.com/mozilla/geckodriver/releases/download/v0.11.1/geckodriver-v0.11.1-linux64.tar.gz";
+  - FILE=`mktemp`; wget "$GECKODRIVER_URL" -qO $FILE && tar xz -C ~/drivers -f $FILE geckodriver; rm $FILE; chmod 777 ~/drivers/geckodriver;
 install:
   - ./manage.sh update_dev_packages
   - pip install coveralls
@@ -19,9 +25,9 @@ script:
   - ./manage.sh styles
   - ./manage.sh grunt_build
   - ./manage.sh tests
-  - ./manage.sh py_test_coverage
 after_success:
-  coveralls
+  - ./manage.sh py_test_coverage
+  - coveralls
 notifications:
   irc:
     channels:

AUTHORS.rst

@@ -8,6 +8,7 @@ Major contributing authors:
 - Alexandre Flament `@dalf <https://github.com/dalf>`_
 - @Cqoicebordel
 - Noémi Ványi
+- Marc Abonce Seguin @a01200356
 
 People who have submitted patches/translates, reported bugs, consulted features or
 generally made searx better:
@@ -43,7 +44,6 @@ generally made searx better:
 - Kang-min Liu
 - Kirill Isakov
 - Guilhem Bonnefille
-- Marc Abonce Seguin
 - @jibe-b
 - Christian Pietsch @pietsch
 - @Maxqia
@@ -55,6 +55,15 @@ generally made searx better:
 - Ammar Najjar @ammarnajjar
 - @stepshal
 - François Revol @mmuman
-- marc @a01200356
 - Harry Wood @harry-wood
 - Thomas Renard @threnard
+- Pydo `<https://github.com/pydo>`_
+- Athemis `<https://github.com/Athemis>`_
+- Stefan Antoni `<http://stefan.antoni.io>`
+- @firebovine
+- Lorenzo J. Lucchini @luccoj
+- @eig8phei
+- Joachim Cherqui
+- @maxigas
+- Jannik Winkel @kiney
+- @juanitobananas

CHANGELOG.rst

@@ -1,3 +1,55 @@
+0.12.0 2017.06.04
+=================
+
+- Python3 compatibility
+- New engines
+
+  - 1337x.to (files, music, video)
+  - Semantic Scholar (science)
+  - Library Genesis (general)
+  - Framalibre (IT)
+  - Free Software Directory (IT)
+- More compact result UI (oscar theme)
+- Configurable static content and template path
+- Spelling suggestions
+- Multiple engine fixes (duckduckgo, bing, swisscows, yahoo news, bing news, twitter, bing images)
+- Reduced static image size
+- Docker updates
+- Translation updates
+
+
+Special thanks to `NLNet <https://nlnet.nl>`__ for sponsoring multiple features of this release.
+
+
+0.11.0 2017.01.10
+=================
+
+- New engines
+
+  - Protein Data Bank Europe (science)
+  - Voat.co (general, social media)
+  - Online Etimology Dictionary (science)
+  - CCC tv (video, it)
+  - Searx (all categories - can rotate multiple other instances)
+- Answerer functionality (see answerer section on /preferences)
+- Local answerers
+
+  - Statistical functions
+  - Random value generator
+- Result proxy support (with `morty <https://github.com/asciimoo/morty>`__)
+- Extended time range filter
+- Improved search language support
+- Multiple engine fixes (digbt, 500px, google news, ixquick, bing, kickass, google play movies, habrahabr, yandex)
+- Minor UI improvements
+- Suggestion support for JSON engine
+- Result and query escaping fixes
+- Configurable HTTP server version
+- More robust search error handling
+- Faster webapp initialization in debug mode
+- Search module refactor
+- Translation updates
+
+
 0.10.0 2016.09.06
 =================
 

Dockerfile

@@ -1,11 +1,11 @@
-FROM alpine:3.3
+FROM alpine:3.5
 MAINTAINER searx <https://github.com/asciimoo/searx>
 LABEL description "A privacy-respecting, hackable metasearch engine."
 
 ENV BASE_URL=False IMAGE_PROXY=False
 EXPOSE 8888
 WORKDIR /usr/local/searx
-CMD ["/usr/bin/tini","--","/usr/local/searx/run.sh"]
+CMD ["/sbin/tini","--","/usr/local/searx/run.sh"]
 
 RUN adduser -D -h /usr/local/searx -s /bin/sh searx searx \
  && echo '#!/bin/sh' >> run.sh \
@@ -36,7 +36,6 @@ RUN echo "@commuedge http://nl.alpinelinux.org/alpine/edge/community" >> /etc/ap
  && apk del \
     build-base \
     python-dev \
-    py-pip\
     libffi-dev \
     openssl-dev \
     libxslt-dev \

manage.sh

@@ -6,12 +6,42 @@ SEARX_DIR="$BASE_DIR/searx"
 ACTION=$1
 
 update_packages() {
-    pip install --upgrade -r "$BASE_DIR/requirements.txt"
+    pip install -r "$BASE_DIR/requirements.txt"
 }
 
 update_dev_packages() {
     update_packages
-    pip install --upgrade -r "$BASE_DIR/requirements-dev.txt"
+    pip install -r "$BASE_DIR/requirements-dev.txt"
+}
+
+check_geckodriver() {
+    echo '[!] Checking geckodriver'
+    set -e
+    geckodriver -V 2>1 > /dev/null || NOTFOUND=1
+    set +e
+    if [ -z $NOTFOUND ]; then
+	return
+    fi
+    GECKODRIVER_VERSION="v0.11.1"
+    PLATFORM=`python -c "import platform; print platform.system().lower(), platform.architecture()[0]"`
+    case $PLATFORM in
+	"linux 32bit" | "linux2 32bit") ARCH="linux32";;
+	"linux 64bit" | "linux2 64bit") ARCH="linux64";;
+	"windows 32 bit") ARCH="win32";;
+	"windows 64 bit") ARCH="win64";;
+	"mac 64bit") ARCH="macos";;
+    esac
+    GECKODRIVER_URL="https://github.com/mozilla/geckodriver/releases/download/$GECKODRIVER_VERSION/geckodriver-$GECKODRIVER_VERSION-$ARCH.tar.gz";
+    if [ -z "$VIRTUAL_ENV" ]; then
+	echo "geckodriver can't be installed because VIRTUAL_ENV is not set, you should download it from\n  $GECKODRIVER_URL"
+	exit
+    else
+	echo "Installing $VIRTUAL_ENV from\n  $GECKODRIVER_URL"
+	FILE=`mktemp`
+	wget "$GECKODRIVER_URL" -qO $FILE && tar xz -C $VIRTUAL_ENV/bin/ -f $FILE geckodriver
+	rm $FILE
+	chmod 777 $VIRTUAL_ENV/bin/geckodriver
+    fi
 }
 
 pep8_check() {
@@ -43,6 +73,7 @@ tests() {
     set -e
     pep8_check
     unit_tests
+    check_geckodriver
     robot_tests
     set +e
 }
@@ -53,8 +84,8 @@ build_style() {
 
 styles() {
     echo '[!] Building styles'
-	build_style themes/default/less/style.less themes/default/css/style.css
-	build_style themes/default/less/style-rtl.less themes/default/css/style-rtl.css
+	build_style themes/legacy/less/style.less themes/legacy/css/style.css
+	build_style themes/legacy/less/style-rtl.less themes/legacy/css/style-rtl.css
 	build_style themes/courgette/less/style.less themes/courgette/css/style.css
 	build_style themes/courgette/less/style-rtl.less themes/courgette/css/style-rtl.css
 	build_style less/bootstrap/bootstrap.less css/bootstrap.min.css
@@ -88,6 +119,7 @@ Commands
     unit_tests           - Run unit tests
     update_dev_packages  - Check & update development and production dependency changes
     update_packages      - Check & update dependency changes
+    check_geckodriver    - Check & download geckodriver (required for robot_tests)
 "
 }
 

requirements-dev.txt

@@ -3,8 +3,8 @@ mock==2.0.0
 nose2[coverage-plugin]
 pep8==1.7.0
 plone.testing==5.0.0
-robotframework-selenium2library==1.7.4
-robotsuite==1.7.0
-transifex-client==0.11
+splinter==0.7.5
+transifex-client==0.12.2
 unittest2==1.1.0
-zope.testrunner==4.4.10
+zope.testrunner==4.5.1
+selenium==3.0.1

requirements.txt

@@ -1,12 +1,10 @@
-certifi==2016.2.28
-flask==0.11.1
+certifi==2017.1.23
+flask==0.12
 flask-babel==0.11.1
-lxml==3.6.0
-ndg-httpsclient==0.4.1
-pyasn1==0.1.9
-pyasn1-modules==0.0.8
+lxml==3.7.3
+idna==2.5
 pygments==2.1.3
-pyopenssl==0.15.1
-python-dateutil==2.5.3
-pyyaml==3.11
-requests[socks]==2.10.0
+pyopenssl==16.2.0
+python-dateutil==2.6.0
+pyyaml==3.12
+requests[socks]==2.13.0

searx/__init__.py

@@ -18,7 +18,7 @@ along with searx. If not, see < http://www.gnu.org/licenses/ >.
 import certifi
 import logging
 from os import environ
-from os.path import realpath, dirname, join, abspath
+from os.path import realpath, dirname, join, abspath, isfile
 from ssl import OPENSSL_VERSION_INFO, OPENSSL_VERSION
 try:
     from yaml import load
@@ -30,25 +30,55 @@ except:
 searx_dir = abspath(dirname(__file__))
 engine_dir = dirname(realpath(__file__))
 
-# if possible set path to settings using the
-# enviroment variable SEARX_SETTINGS_PATH
+
+def check_settings_yml(file_name):
+    if isfile(file_name):
+        return file_name
+    else:
+        return None
+
+# find location of settings.yml
 if 'SEARX_SETTINGS_PATH' in environ:
-    settings_path = environ['SEARX_SETTINGS_PATH']
-# otherwise using default path
+    # if possible set path to settings using the
+    # enviroment variable SEARX_SETTINGS_PATH
+    settings_path = check_settings_yml(environ['SEARX_SETTINGS_PATH'])
 else:
-    settings_path = join(searx_dir, 'settings.yml')
+    # if not, get it from searx code base or last solution from /etc/searx
+    settings_path = check_settings_yml(join(searx_dir, 'settings.yml')) or check_settings_yml('/etc/searx/settings.yml')
+
+if not settings_path:
+    raise Exception('settings.yml not found')
 
 # load settings
 with open(settings_path) as settings_yaml:
     settings = load(settings_yaml)
 
-if settings.get('general', {}).get('debug'):
+'''
+enable debug if
+the environnement variable SEARX_DEBUG is 1 or true
+(whatever the value in settings.yml)
+or general.debug=True in settings.yml
+
+disable debug if
+the environnement variable SEARX_DEBUG is 0 or false
+(whatever the value in settings.yml)
+or general.debug=False in settings.yml
+'''
+searx_debug_env = environ.get('SEARX_DEBUG', '').lower()
+if searx_debug_env == 'true' or searx_debug_env == '1':
+    searx_debug = True
+elif searx_debug_env == 'false' or searx_debug_env == '0':
+    searx_debug = False
+else:
+    searx_debug = settings.get('general', {}).get('debug')
+
+if searx_debug:
     logging.basicConfig(level=logging.DEBUG)
 else:
     logging.basicConfig(level=logging.WARNING)
 
 logger = logging.getLogger('searx')
-
+logger.debug('read configuration from %s', settings_path)
 # Workaround for openssl versions <1.0.2
 # https://github.com/certifi/python-certifi/issues/26
 if OPENSSL_VERSION_INFO[0:3] < (1, 0, 2):

searx/answerers/__init__.py

@@ -0,0 +1,50 @@
+from os import listdir
+from os.path import realpath, dirname, join, isdir
+from sys import version_info
+from searx.utils import load_module
+from collections import defaultdict
+
+if version_info[0] == 3:
+    unicode = str
+
+
+answerers_dir = dirname(realpath(__file__))
+
+
+def load_answerers():
+    answerers = []
+    for filename in listdir(answerers_dir):
+        if not isdir(join(answerers_dir, filename)) or filename.startswith('_'):
+            continue
+        module = load_module('answerer.py', join(answerers_dir, filename))
+        if not hasattr(module, 'keywords') or not isinstance(module.keywords, tuple) or not len(module.keywords):
+            exit(2)
+        answerers.append(module)
+    return answerers
+
+
+def get_answerers_by_keywords(answerers):
+    by_keyword = defaultdict(list)
+    for answerer in answerers:
+        for keyword in answerer.keywords:
+            for keyword in answerer.keywords:
+                by_keyword[keyword].append(answerer.answer)
+    return by_keyword
+
+
+def ask(query):
+    results = []
+    query_parts = list(filter(None, query.query.split()))
+
+    if query_parts[0].decode('utf-8') not in answerers_by_keywords:
+        return results
+
+    for answerer in answerers_by_keywords[query_parts[0].decode('utf-8')]:
+        result = answerer(query)
+        if result:
+            results.append(result)
+    return results
+
+
+answerers = load_answerers()
+answerers_by_keywords = get_answerers_by_keywords(answerers)

searx/answerers/random/answerer.py

@@ -0,0 +1,55 @@
+import random
+import string
+import sys
+from flask_babel import gettext
+
+# required answerer attribute
+# specifies which search query keywords triggers this answerer
+keywords = ('random',)
+
+random_int_max = 2**31
+
+if sys.version_info[0] == 2:
+    random_string_letters = string.lowercase + string.digits + string.uppercase
+else:
+    unicode = str
+    random_string_letters = string.ascii_lowercase + string.digits + string.ascii_uppercase
+
+
+def random_string():
+    return u''.join(random.choice(random_string_letters)
+                    for _ in range(random.randint(8, 32)))
+
+
+def random_float():
+    return unicode(random.random())
+
+
+def random_int():
+    return unicode(random.randint(-random_int_max, random_int_max))
+
+
+random_types = {b'string': random_string,
+                b'int': random_int,
+                b'float': random_float}
+
+
+# required answerer function
+# can return a list of results (any result type) for a given query
+def answer(query):
+    parts = query.query.split()
+    if len(parts) != 2:
+        return []
+
+    if parts[1] not in random_types:
+        return []
+
+    return [{'answer': random_types[parts[1]]()}]
+
+
+# required answerer function
+# returns information about the answerer
+def self_info():
+    return {'name': gettext('Random value generator'),
+            'description': gettext('Generate different random values'),
+            'examples': [u'random {}'.format(x) for x in random_types]}

searx/answerers/statistics/answerer.py

@@ -0,0 +1,55 @@
+from sys import version_info
+from functools import reduce
+from operator import mul
+
+from flask_babel import gettext
+
+if version_info[0] == 3:
+    unicode = str
+
+keywords = ('min',
+            'max',
+            'avg',
+            'sum',
+            'prod')
+
+
+# required answerer function
+# can return a list of results (any result type) for a given query
+def answer(query):
+    parts = query.query.split()
+
+    if len(parts) < 2:
+        return []
+
+    try:
+        args = list(map(float, parts[1:]))
+    except:
+        return []
+
+    func = parts[0]
+    answer = None
+
+    if func == b'min':
+        answer = min(args)
+    elif func == b'max':
+        answer = max(args)
+    elif func == b'avg':
+        answer = sum(args) / len(args)
+    elif func == b'sum':
+        answer = sum(args)
+    elif func == b'prod':
+        answer = reduce(mul, args, 1)
+
+    if answer is None:
+        return []
+
+    return [{'answer': unicode(answer)}]
+
+
+# required answerer function
+# returns information about the answerer
+def self_info():
+    return {'name': gettext('Statistics functions'),
+            'description': gettext('Compute {functions} of the arguments').format(functions='/'.join(keywords)),
+            'examples': ['avg 123 548 2.04 24.2']}

searx/autocomplete.py

@@ -18,7 +18,6 @@ along with searx. If not, see < http://www.gnu.org/licenses/ >.
 
 from lxml import etree
 from json import loads
-from urllib import urlencode
 from searx import settings
 from searx.languages import language_codes
 from searx.engines import (
@@ -26,6 +25,11 @@ from searx.engines import (
 )
 from searx.poolrequests import get as http_get
 
+try:
+    from urllib import urlencode
+except:
+    from urllib.parse import urlencode
+
 
 def get(*args, **kwargs):
     if 'timeout' not in kwargs:
@@ -81,22 +85,22 @@ def searx_bang(full_query):
             engine_query = full_query.getSearchQuery()[1:]
 
             for lc in language_codes:
-                lang_id, lang_name, country = map(str.lower, lc)
+                lang_id, lang_name, country, english_name = map(unicode.lower, lc)
 
                 # check if query starts with language-id
                 if lang_id.startswith(engine_query):
                     if len(engine_query) <= 2:
-                        results.append(':{lang_id}'.format(lang_id=lang_id.split('_')[0]))
+                        results.append(u':{lang_id}'.format(lang_id=lang_id.split('-')[0]))
                     else:
-                        results.append(':{lang_id}'.format(lang_id=lang_id))
+                        results.append(u':{lang_id}'.format(lang_id=lang_id))
 
                 # check if query starts with language name
-                if lang_name.startswith(engine_query):
-                    results.append(':{lang_name}'.format(lang_name=lang_name))
+                if lang_name.startswith(engine_query) or english_name.startswith(engine_query):
+                    results.append(u':{lang_name}'.format(lang_name=lang_name))
 
                 # check if query starts with country
                 if country.startswith(engine_query.replace('_', ' ')):
-                    results.append(':{country}'.format(country=country.replace(' ', '_')))
+                    results.append(u':{country}'.format(country=country.replace(' ', '_')))
 
     # remove duplicates
     result_set = set(results)

searx/engines/1337x.py

@@ -0,0 +1,39 @@
+from lxml import html
+from searx.engines.xpath import extract_text
+from searx.utils import get_torrent_size
+from searx.url_utils import quote, urljoin
+
+url = 'https://1337x.to/'
+search_url = url + 'search/{search_term}/{pageno}/'
+categories = ['videos']
+paging = True
+
+
+def request(query, params):
+    params['url'] = search_url.format(search_term=quote(query), pageno=params['pageno'])
+
+    return params
+
+
+def response(resp):
+    results = []
+
+    dom = html.fromstring(resp.text)
+
+    for result in dom.xpath('//table[contains(@class, "table-list")]/tbody//tr'):
+        href = urljoin(url, result.xpath('./td[contains(@class, "name")]/a[2]/@href')[0])
+        title = extract_text(result.xpath('./td[contains(@class, "name")]/a[2]'))
+        seed = extract_text(result.xpath('.//td[contains(@class, "seeds")]'))
+        leech = extract_text(result.xpath('.//td[contains(@class, "leeches")]'))
+        filesize_info = extract_text(result.xpath('.//td[contains(@class, "size")]/text()'))
+        filesize, filesize_multiplier = filesize_info.split()
+        filesize = get_torrent_size(filesize, filesize_multiplier)
+
+        results.append({'url': href,
+                        'title': title,
+                        'seed': seed,
+                        'leech': leech,
+                        'filesize': filesize,
+                        'template': 'torrent.html'})
+
+    return results

searx/engines/__init__.py

@@ -16,13 +16,15 @@ along with searx. If not, see < http://www.gnu.org/licenses/ >.
 (C) 2013- by Adam Tauber, <asciimoo@gmail.com>
 '''
 
-from os.path import realpath, dirname, splitext, join
+from os.path import realpath, dirname
 import sys
-from imp import load_source
 from flask_babel import gettext
 from operator import itemgetter
+from json import loads
+from requests import get
 from searx import settings
 from searx import logger
+from searx.utils import load_module
 
 
 logger = logger.getChild('engines')
@@ -33,10 +35,13 @@ engines = {}
 
 categories = {'general': []}
 
+languages = loads(open(engine_dir + '/../data/engines_languages.json').read())
+
 engine_shortcuts = {}
 engine_default_args = {'paging': False,
                        'categories': ['general'],
                        'language_support': True,
+                       'supported_languages': [],
                        'safesearch': False,
                        'timeout': settings['outgoing']['request_timeout'],
                        'shortcut': '-',
@@ -46,22 +51,18 @@ engine_default_args = {'paging': False,
                        'time_range_support': False}
 
 
-def load_module(filename):
-    modname = splitext(filename)[0]
-    if modname in sys.modules:
-        del sys.modules[modname]
-    filepath = join(engine_dir, filename)
-    module = load_source(modname, filepath)
-    module.name = modname
-    return module
-
-
 def load_engine(engine_data):
-    engine_name = engine_data['engine']
+
+    if '_' in engine_data['name']:
+        logger.error('Engine name conains underscore: "{}"'.format(engine_data['name']))
+        sys.exit(1)
+
+    engine_module = engine_data['engine']
+
     try:
-        engine = load_module(engine_name + '.py')
+        engine = load_module(engine_module + '.py', engine_dir)
     except:
-        logger.exception('Cannot load engine "{}"'.format(engine_name))
+        logger.exception('Cannot load engine "{}"'.format(engine_module))
         return None
 
     for param_name in engine_data:
@@ -71,12 +72,11 @@ def load_engine(engine_data):
             if engine_data['categories'] == 'none':
                 engine.categories = []
             else:
-                engine.categories = map(
-                    str.strip, engine_data['categories'].split(','))
+                engine.categories = list(map(str.strip, engine_data['categories'].split(',')))
             continue
         setattr(engine, param_name, engine_data[param_name])
 
-    for arg_name, arg_value in engine_default_args.iteritems():
+    for arg_name, arg_value in engine_default_args.items():
         if not hasattr(engine, arg_name):
             setattr(engine, arg_name, arg_value)
 
@@ -89,10 +89,22 @@ def load_engine(engine_data):
                          .format(engine.name, engine_attr))
             sys.exit(1)
 
+    # assign supported languages from json file
+    if engine_data['name'] in languages:
+        setattr(engine, 'supported_languages', languages[engine_data['name']])
+
+    # assign language fetching method if auxiliary method exists
+    if hasattr(engine, '_fetch_supported_languages'):
+        setattr(engine, 'fetch_supported_languages',
+                lambda: engine._fetch_supported_languages(get(engine.supported_languages_url)))
+
     engine.stats = {
         'result_count': 0,
         'search_count': 0,
         'page_load_time': 0,
+        'page_load_count': 0,
+        'engine_time': 0,
+        'engine_time_count': 0,
         'score_count': 0,
         'errors': 0
     }
@@ -109,32 +121,56 @@ def load_engine(engine_data):
     return engine
 
 
+def to_percentage(stats, maxvalue):
+    for engine_stat in stats:
+        if maxvalue:
+            engine_stat['percentage'] = int(engine_stat['avg'] / maxvalue * 100)
+        else:
+            engine_stat['percentage'] = 0
+    return stats
+
+
 def get_engines_stats():
     # TODO refactor
     pageloads = []
+    engine_times = []
     results = []
     scores = []
     errors = []
     scores_per_result = []
 
-    max_pageload = max_results = max_score = max_errors = max_score_per_result = 0  # noqa
+    max_pageload = max_engine_times = max_results = max_score = max_errors = max_score_per_result = 0  # noqa
     for engine in engines.values():
         if engine.stats['search_count'] == 0:
             continue
         results_num = \
             engine.stats['result_count'] / float(engine.stats['search_count'])
-        load_times = engine.stats['page_load_time'] / float(engine.stats['search_count'])  # noqa
+
+        if engine.stats['page_load_count'] != 0:
+            load_times = engine.stats['page_load_time'] / float(engine.stats['page_load_count'])  # noqa
+        else:
+            load_times = 0
+
+        if engine.stats['engine_time_count'] != 0:
+            this_engine_time = engine.stats['engine_time'] / float(engine.stats['engine_time_count'])  # noqa
+        else:
+            this_engine_time = 0
+
         if results_num:
             score = engine.stats['score_count'] / float(engine.stats['search_count'])  # noqa
             score_per_result = score / results_num
         else:
             score = score_per_result = 0.0
-        max_results = max(results_num, max_results)
+
         max_pageload = max(load_times, max_pageload)
+        max_engine_times = max(this_engine_time, max_engine_times)
+        max_results = max(results_num, max_results)
         max_score = max(score, max_score)
         max_score_per_result = max(score_per_result, max_score_per_result)
         max_errors = max(max_errors, engine.stats['errors'])
+
         pageloads.append({'avg': load_times, 'name': engine.name})
+        engine_times.append({'avg': this_engine_time, 'name': engine.name})
         results.append({'avg': results_num, 'name': engine.name})
         scores.append({'avg': score, 'name': engine.name})
         errors.append({'avg': engine.stats['errors'], 'name': engine.name})
@@ -143,38 +179,18 @@ def get_engines_stats():
             'name': engine.name
         })
 
-    for engine in pageloads:
-        if max_pageload:
-            engine['percentage'] = int(engine['avg'] / max_pageload * 100)
-        else:
-            engine['percentage'] = 0
-
-    for engine in results:
-        if max_results:
-            engine['percentage'] = int(engine['avg'] / max_results * 100)
-        else:
-            engine['percentage'] = 0
-
-    for engine in scores:
-        if max_score:
-            engine['percentage'] = int(engine['avg'] / max_score * 100)
-        else:
-            engine['percentage'] = 0
-
-    for engine in scores_per_result:
-        if max_score_per_result:
-            engine['percentage'] = int(engine['avg']
-                                       / max_score_per_result * 100)
-        else:
-            engine['percentage'] = 0
-
-    for engine in errors:
-        if max_errors:
-            engine['percentage'] = int(float(engine['avg']) / max_errors * 100)
-        else:
-            engine['percentage'] = 0
+    pageloads = to_percentage(pageloads, max_pageload)
+    engine_times = to_percentage(engine_times, max_engine_times)
+    results = to_percentage(results, max_results)
+    scores = to_percentage(scores, max_score)
+    scores_per_result = to_percentage(scores_per_result, max_score_per_result)
+    erros = to_percentage(errors, max_errors)
 
     return [
+        (
+            gettext('Engine time (sec)'),
+            sorted(engine_times, key=itemgetter('avg'))
+        ),
         (
             gettext('Page loads (sec)'),
             sorted(pageloads, key=itemgetter('avg'))
@@ -198,11 +214,8 @@ def get_engines_stats():
     ]
 
 
-if 'engines' not in settings or not settings['engines']:
-    logger.error('No engines found. Edit your settings.yml')
-    exit(2)
-
-for engine_data in settings['engines']:
-    engine = load_engine(engine_data)
-    if engine is not None:
-        engines[engine.name] = engine
+def initialize_engines(engine_list):
+    for engine_data in engine_list:
+        engine = load_engine(engine_data)
+        if engine is not None:
+            engines[engine.name] = engine

searx/engines/archlinux.py

@@ -11,11 +11,9 @@
  @parse        url, title
 """
 
-from urlparse import urljoin
-from cgi import escape
-from urllib import urlencode
 from lxml import html
 from searx.engines.xpath import extract_text
+from searx.url_utils import urlencode, urljoin
 
 # engine dependent config
 categories = ['it']
@@ -30,8 +28,8 @@ xpath_link = './/div[@class="mw-search-result-heading"]/a'
 
 # cut 'en' from 'en_US', 'de' from 'de_CH', and so on
 def locale_to_lang_code(locale):
-    if locale.find('_') >= 0:
-        locale = locale.split('_')[0]
+    if locale.find('-') >= 0:
+        locale = locale.split('-')[0]
     return locale
 
 
@@ -96,6 +94,7 @@ main_langs = {
     'uk': 'Українська',
     'zh': '简体中文'
 }
+supported_languages = dict(lang_urls, **main_langs)
 
 
 # do search-request
@@ -135,7 +134,7 @@ def response(resp):
     for result in dom.xpath(xpath_results):
         link = result.xpath(xpath_link)[0]
         href = urljoin(base_url, link.attrib.get('href'))
-        title = escape(extract_text(link))
+        title = extract_text(link)
 
         results.append({'url': href,
                         'title': title})

searx/engines/base.py

@@ -14,11 +14,10 @@
 """
 
 from lxml import etree
-from urllib import urlencode
-from searx.utils import searx_useragent
-from cgi import escape
 from datetime import datetime
 import re
+from searx.url_utils import urlencode
+from searx.utils import searx_useragent
 
 
 categories = ['science']
@@ -74,7 +73,7 @@ def request(query, params):
 def response(resp):
     results = []
 
-    search_results = etree.XML(resp.content)
+    search_results = etree.XML(resp.text)
 
     for entry in search_results.xpath('./result/doc'):
         content = "No description available"
@@ -94,7 +93,7 @@ def response(resp):
                 url = item.text
 
             elif item.attrib["name"] == "dcdescription":
-                content = escape(item.text[:300])
+                content = item.text[:300]
                 if len(item.text) > 300:
                     content += "..."
 

searx/engines/bing.py

@@ -13,15 +13,15 @@
  @todo        publishedDate
 """
 
-from urllib import urlencode
-from cgi import escape
 from lxml import html
 from searx.engines.xpath import extract_text
+from searx.url_utils import urlencode
 
 # engine dependent config
 categories = ['general']
 paging = True
 language_support = True
+supported_languages_url = 'https://www.bing.com/account/general'
 
 # search-url
 base_url = 'https://www.bing.com/'
@@ -32,18 +32,17 @@ search_string = 'search?{query}&first={offset}'
 def request(query, params):
     offset = (params['pageno'] - 1) * 10 + 1
 
-    if params['language'] == 'all':
-        language = 'en-US'
+    if params['language'] != 'all':
+        lang = params['language'].split('-')[0].upper()
     else:
-        language = params['language'].replace('_', '-')
+        lang = 'EN'
+
+    query = u'language:{} {}'.format(lang, query.decode('utf-8')).encode('utf-8')
 
     search_path = search_string.format(
-        query=urlencode({'q': query, 'setmkt': language}),
+        query=urlencode({'q': query}),
         offset=offset)
 
-    params['cookies']['SRCHHPGUSR'] = \
-        'NEWWND=0&NRSLT=-1&SRCHLANG=' + language.split('-')[0]
-
     params['url'] = base_url + search_path
     return params
 
@@ -65,7 +64,7 @@ def response(resp):
         link = result.xpath('.//h3/a')[0]
         url = link.attrib.get('href')
         title = extract_text(link)
-        content = escape(extract_text(result.xpath('.//p')))
+        content = extract_text(result.xpath('.//p'))
 
         # append result
         results.append({'url': url,
@@ -77,7 +76,7 @@ def response(resp):
         link = result.xpath('.//h2/a')[0]
         url = link.attrib.get('href')
         title = extract_text(link)
-        content = escape(extract_text(result.xpath('.//p')))
+        content = extract_text(result.xpath('.//p'))
 
         # append result
         results.append({'url': url,
@@ -86,3 +85,17 @@ def response(resp):
 
     # return results
     return results
+
+
+# get supported languages from their site
+def _fetch_supported_languages(resp):
+    supported_languages = []
+    dom = html.fromstring(resp.text)
+    options = dom.xpath('//div[@id="limit-languages"]//input')
+    for option in options:
+        code = option.xpath('./@id')[0].replace('_', '-')
+        if code == 'nb':
+            code = 'no'
+        supported_languages.append(code)
+
+    return supported_languages

searx/engines/bing_images.py

@@ -15,20 +15,26 @@
               limited response to 10 images
 """
 
-from urllib import urlencode
 from lxml import html
 from json import loads
 import re
+from searx.engines.bing import _fetch_supported_languages, supported_languages_url
+from searx.url_utils import urlencode
 
 # engine dependent config
 categories = ['images']
 paging = True
 safesearch = True
+time_range_support = True
 
 # search-url
 base_url = 'https://www.bing.com/'
 search_string = 'images/search?{query}&count=10&first={offset}'
-thumb_url = "https://www.bing.com/th?id={ihk}"
+time_range_string = '&qft=+filterui:age-lt{interval}'
+time_range_dict = {'day': '1440',
+                   'week': '10080',
+                   'month': '43200',
+                   'year': '525600'}
 
 # safesearch definitions
 safesearch_types = {2: 'STRICT',
@@ -47,7 +53,7 @@ def request(query, params):
     if params['language'] == 'all':
         language = 'en-US'
     else:
-        language = params['language'].replace('_', '-')
+        language = params['language']
 
     search_path = search_string.format(
         query=urlencode({'q': query}),
@@ -58,6 +64,8 @@ def request(query, params):
         '&ADLT=' + safesearch_types.get(params['safesearch'], 'DEMOTE')
 
     params['url'] = base_url + search_path
+    if params['time_range'] in time_range_dict:
+        params['url'] += time_range_string.format(interval=time_range_dict[params['time_range']])
 
     return params
 
@@ -69,30 +77,32 @@ def response(resp):
     dom = html.fromstring(resp.text)
 
     # parse results
-    for result in dom.xpath('//div[@class="dg_u"]/div'):
+    for result in dom.xpath('//div[@id="mmComponent_images_1"]/ul/li/div/div[@class="imgpt"]'):
         link = result.xpath('./a')[0]
 
+        # TODO find actual title
+        title = link.xpath('.//img/@alt')[0]
+
         # parse json-data (it is required to add a space, to make it parsable)
         json_data = loads(_quote_keys_regex.sub(r'\1"\2": \3', link.attrib.get('m')))
 
-        title = link.attrib.get('t1')
-        ihk = link.attrib.get('ihk')
+        url = json_data.get('purl')
+        img_src = json_data.get('murl')
 
-        # url = 'http://' + link.attrib.get('t3')
-        url = json_data.get('surl')
-        img_src = json_data.get('imgurl')
+        thumb_json_data = loads(_quote_keys_regex.sub(r'\1"\2": \3', link.attrib.get('mad')))
+        thumbnail = thumb_json_data.get('turl')
 
         # append result
         results.append({'template': 'images.html',
                         'url': url,
                         'title': title,
                         'content': '',
-                        'thumbnail_src': thumb_url.format(ihk=ihk),
+                        'thumbnail_src': thumbnail,
                         'img_src': img_src})
 
         # TODO stop parsing if 10 images are found
-        if len(results) >= 10:
-            break
+        # if len(results) >= 10:
+        #     break
 
     # return results
     return results

searx/engines/bing_news.py

@@ -11,21 +11,26 @@
  @parse       url, title, content, publishedDate, thumbnail
 """
 
-from urllib import urlencode
-from urlparse import urlparse, parse_qsl
 from datetime import datetime
 from dateutil import parser
 from lxml import etree
 from searx.utils import list_get
+from searx.engines.bing import _fetch_supported_languages, supported_languages_url
+from searx.url_utils import urlencode, urlparse, parse_qsl
 
 # engine dependent config
 categories = ['news']
 paging = True
 language_support = True
+time_range_support = True
 
 # search-url
 base_url = 'https://www.bing.com/'
 search_string = 'news/search?{query}&first={offset}&format=RSS'
+search_string_with_time = 'news/search?{query}&first={offset}&qft=interval%3d"{interval}"&format=RSS'
+time_range_dict = {'day': '7',
+                   'week': '8',
+                   'month': '9'}
 
 
 # remove click
@@ -46,20 +51,32 @@ def image_url_cleanup(url_string):
     return url_string
 
 
+def _get_url(query, language, offset, time_range):
+    if time_range in time_range_dict:
+        search_path = search_string_with_time.format(
+            query=urlencode({'q': query, 'setmkt': language}),
+            offset=offset,
+            interval=time_range_dict[time_range])
+    else:
+        search_path = search_string.format(
+            query=urlencode({'q': query, 'setmkt': language}),
+            offset=offset)
+    return base_url + search_path
+
+
 # do search-request
 def request(query, params):
+    if params['time_range'] and params['time_range'] not in time_range_dict:
+        return params
+
     offset = (params['pageno'] - 1) * 10 + 1
 
     if params['language'] == 'all':
         language = 'en-US'
     else:
-        language = params['language'].replace('_', '-')
+        language = params['language']
 
-    search_path = search_string.format(
-        query=urlencode({'q': query, 'setmkt': language}),
-        offset=offset)
-
-    params['url'] = base_url + search_path
+    params['url'] = _get_url(query, language, offset, params['time_range'])
 
     return params
 
@@ -95,12 +112,11 @@ def response(resp):
 
         # append result
         if thumbnail is not None:
-            results.append({'template': 'videos.html',
-                            'url': url,
+            results.append({'url': url,
                             'title': title,
                             'publishedDate': publishedDate,
                             'content': content,
-                            'thumbnail': thumbnail})
+                            'img_src': thumbnail})
         else:
             results.append({'url': url,
                             'title': title,

searx/engines/blekko_images.py

@@ -11,7 +11,7 @@
 """
 
 from json import loads
-from urllib import urlencode
+from searx.url_utils import urlencode
 
 # engine dependent config
 categories = ['images']

searx/engines/btdigg.py

@@ -10,12 +10,10 @@
  @parse       url, title, content, seed, leech, magnetlink
 """
 
-from urlparse import urljoin
-from cgi import escape
-from urllib import quote
 from lxml import html
 from operator import itemgetter
 from searx.engines.xpath import extract_text
+from searx.url_utils import quote, urljoin
 from searx.utils import get_torrent_size
 
 # engine dependent config
@@ -39,7 +37,7 @@ def request(query, params):
 def response(resp):
     results = []
 
-    dom = html.fromstring(resp.content)
+    dom = html.fromstring(resp.text)
 
     search_res = dom.xpath('//div[@id="search_res"]/table/tr')
 
@@ -51,8 +49,8 @@ def response(resp):
     for result in search_res:
         link = result.xpath('.//td[@class="torrent_name"]//a')[0]
         href = urljoin(url, link.attrib.get('href'))
-        title = escape(extract_text(link))
-        content = escape(extract_text(result.xpath('.//pre[@class="snippet"]')[0]))
+        title = extract_text(link)
+        content = extract_text(result.xpath('.//pre[@class="snippet"]')[0])
         content = "<br />".join(content.split("\n"))
 
         filesize = result.xpath('.//span[@class="attr_val"]/text()')[0].split()[0]

searx/engines/currency_convert.py

@@ -1,21 +1,25 @@
-from datetime import datetime
+import json
 import re
 import os
-import json
+import sys
 import unicodedata
 
+from datetime import datetime
+
+if sys.version_info[0] == 3:
+    unicode = str
 
 categories = []
 url = 'https://download.finance.yahoo.com/d/quotes.csv?e=.csv&f=sl1d1t1&s={query}=X'
 weight = 100
 
-parser_re = re.compile(u'.*?(\\d+(?:\\.\\d+)?) ([^.0-9]+) (?:in|to) ([^.0-9]+)', re.I)  # noqa
+parser_re = re.compile(b'.*?(\\d+(?:\\.\\d+)?) ([^.0-9]+) (?:in|to) ([^.0-9]+)', re.I)
 
 db = 1
 
 
 def normalize_name(name):
-    name = name.lower().replace('-', ' ').rstrip('s')
+    name = name.decode('utf-8').lower().replace('-', ' ').rstrip('s')
     name = re.sub(' +', ' ', name)
     return unicodedata.normalize('NFKD', name).lower()
 
@@ -35,7 +39,7 @@ def iso4217_to_name(iso4217, language):
 
 
 def request(query, params):
-    m = parser_re.match(unicode(query, 'utf8'))
+    m = parser_re.match(query)
     if not m:
         # wrong query
         return params

searx/engines/dailymotion.py

@@ -12,10 +12,9 @@
  @todo        set content-parameter with correct data
 """
 
-from urllib import urlencode
 from json import loads
-from cgi import escape
 from datetime import datetime
+from searx.url_utils import urlencode
 
 # engine dependent config
 categories = ['videos']
@@ -28,6 +27,8 @@ search_url = 'https://api.dailymotion.com/videos?fields=created_time,title,descr
 embedded_url = '<iframe frameborder="0" width="540" height="304" ' +\
     'data-src="//www.dailymotion.com/embed/video/{videoid}" allowfullscreen></iframe>'
 
+supported_languages_url = 'https://api.dailymotion.com/languages'
+
 
 # do search-request
 def request(query, params):
@@ -57,7 +58,7 @@ def response(resp):
     for res in search_res['list']:
         title = res['title']
         url = res['url']
-        content = escape(res['description'])
+        content = res['description']
         thumbnail = res['thumbnail_360_url']
         publishedDate = datetime.fromtimestamp(res['created_time'], None)
         embedded = embedded_url.format(videoid=res['id'])
@@ -75,3 +76,22 @@ def response(resp):
 
     # return results
     return results
+
+
+# get supported languages from their site
+def _fetch_supported_languages(resp):
+    supported_languages = {}
+
+    response_json = loads(resp.text)
+
+    for language in response_json['list']:
+        supported_languages[language['code']] = {}
+
+        name = language['native_name']
+        if name:
+            supported_languages[language['code']]['name'] = name
+        english_name = language['name']
+        if english_name:
+            supported_languages[language['code']]['english_name'] = english_name
+
+    return supported_languages

searx/engines/deezer.py

@@ -11,7 +11,7 @@
 """
 
 from json import loads
-from urllib import urlencode
+from searx.url_utils import urlencode
 
 # engine dependent config
 categories = ['music']
@@ -30,8 +30,7 @@ embedded_url = '<iframe scrolling="no" frameborder="0" allowTransparency="true"
 def request(query, params):
     offset = (params['pageno'] - 1) * 25
 
-    params['url'] = search_url.format(query=urlencode({'q': query}),
-                                      offset=offset)
+    params['url'] = search_url.format(query=urlencode({'q': query}), offset=offset)
 
     return params
 
@@ -51,10 +50,11 @@ def response(resp):
             if url.startswith('http://'):
                 url = 'https' + url[4:]
 
-            content = result['artist']['name'] +\
-                " &bull; " +\
-                result['album']['title'] +\
-                " &bull; " + result['title']
+            content = u'{} - {} - {}'.format(
+                result['artist']['name'],
+                result['album']['title'],
+                result['title'])
+
             embedded = embedded_url.format(audioid=result['id'])
 
             # append result

searx/engines/deviantart.py

@@ -12,10 +12,10 @@
  @todo        rewrite to api
 """
 
-from urllib import urlencode
 from lxml import html
 import re
 from searx.engines.xpath import extract_text
+from searx.url_utils import urlencode
 
 # engine dependent config
 categories = ['images']
@@ -34,6 +34,9 @@ time_range_dict = {'day': 11,
 
 # do search-request
 def request(query, params):
+    if params['time_range'] and params['time_range'] not in time_range_dict:
+        return params
+
     offset = (params['pageno'] - 1) * 24
 
     params['url'] = search_url.format(offset=offset,

searx/engines/dictzone.py

@@ -10,21 +10,20 @@
 """
 
 import re
-from urlparse import urljoin
 from lxml import html
-from cgi import escape
 from searx.utils import is_valid_lang
+from searx.url_utils import urljoin
 
 categories = ['general']
 url = u'http://dictzone.com/{from_lang}-{to_lang}-dictionary/{query}'
 weight = 100
 
-parser_re = re.compile(u'.*?([a-z]+)-([a-z]+) ([^ ]+)$', re.I)
+parser_re = re.compile(b'.*?([a-z]+)-([a-z]+) ([^ ]+)$', re.I)
 results_xpath = './/table[@id="r"]/tr'
 
 
 def request(query, params):
-    m = parser_re.match(unicode(query, 'utf8'))
+    m = parser_re.match(query)
     if not m:
         return params
 
@@ -62,8 +61,8 @@ def response(resp):
 
         results.append({
             'url': urljoin(resp.url, '?%d' % k),
-            'title': escape(from_result.text_content()),
-            'content': escape('; '.join(to_results))
+            'title': from_result.text_content(),
+            'content': '; '.join(to_results)
         })
 
     return results

searx/engines/digbt.py

@@ -10,10 +10,14 @@
  @parse       url, title, content, magnetlink
 """
 
-from urlparse import urljoin
+from sys import version_info
 from lxml import html
 from searx.engines.xpath import extract_text
 from searx.utils import get_torrent_size
+from searx.url_utils import urljoin
+
+if version_info[0] == 3:
+    unicode = str
 
 categories = ['videos', 'music', 'files']
 paging = True
@@ -31,7 +35,7 @@ def request(query, params):
 
 
 def response(resp):
-    dom = html.fromstring(resp.content)
+    dom = html.fromstring(resp.text)
     search_res = dom.xpath('.//td[@class="x-item"]')
 
     if not search_res:
@@ -40,7 +44,7 @@ def response(resp):
     results = list()
     for result in search_res:
         url = urljoin(URL, result.xpath('.//a[@title]/@href')[0])
-        title = result.xpath('.//a[@title]/text()')[0]
+        title = extract_text(result.xpath('.//a[@title]'))
         content = extract_text(result.xpath('.//div[@class="files"]'))
         files_data = extract_text(result.xpath('.//div[@class="tail"]')).split()
         filesize = get_torrent_size(files_data[FILESIZE], files_data[FILESIZE_MULTIPLIER])

searx/engines/digg.py

@@ -10,11 +10,10 @@
  @parse       url, title, content, publishedDate, thumbnail
 """
 
-from urllib import quote_plus
+from dateutil import parser
 from json import loads
 from lxml import html
-from cgi import escape
-from dateutil import parser
+from searx.url_utils import quote_plus
 
 # engine dependent config
 categories = ['news', 'social media']
@@ -56,7 +55,7 @@ def response(resp):
         url = result.attrib.get('data-contenturl')
         thumbnail = result.xpath('.//img')[0].attrib.get('src')
         title = ''.join(result.xpath(title_xpath))
-        content = escape(''.join(result.xpath(content_xpath)))
+        content = ''.join(result.xpath(content_xpath))
         pubdate = result.xpath(pubdate_xpath)[0].attrib.get('datetime')
         publishedDate = parser.parse(pubdate)
 

searx/engines/doku.py

@@ -9,9 +9,9 @@
 # @stable      yes
 # @parse       (general)    url, title, content
 
-from urllib import urlencode
 from lxml.html import fromstring
 from searx.engines.xpath import extract_text
+from searx.url_utils import urlencode
 
 # engine dependent config
 categories = ['general']  # TODO , 'images', 'music', 'videos', 'files'

searx/engines/duckduckgo.py

@@ -13,19 +13,21 @@
  @todo        rewrite to api
 """
 
-from urllib import urlencode
 from lxml.html import fromstring
+from requests import get
+from json import loads
 from searx.engines.xpath import extract_text
-from searx.languages import language_codes
+from searx.url_utils import urlencode
 
 # engine dependent config
 categories = ['general']
 paging = True
 language_support = True
+supported_languages_url = 'https://duckduckgo.com/d2030.js'
 time_range_support = True
 
 # search-url
-url = 'https://duckduckgo.com/html?{query}&s={offset}'
+url = 'https://duckduckgo.com/html?{query}&s={offset}&api=/d.js&o=json&dc={dc_param}'
 time_range_url = '&df={range}'
 
 time_range_dict = {'day': 'd',
@@ -39,33 +41,53 @@ title_xpath = './/a[@class="result__a"]'
 content_xpath = './/a[@class="result__snippet"]'
 
 
-# do search-request
-def request(query, params):
-    offset = (params['pageno'] - 1) * 30
-
-    if params['language'] == 'all':
-        locale = None
+# match query's language to a region code that duckduckgo will accept
+def get_region_code(lang):
+    # custom fixes for languages
+    if lang == 'all':
+        region_code = None
+    elif lang[:2] == 'ja':
+        region_code = 'jp-jp'
+    elif lang[:2] == 'sl':
+        region_code = 'sl-sl'
+    elif lang == 'zh-TW':
+        region_code = 'tw-tzh'
+    elif lang == 'zh-HK':
+        region_code = 'hk-tzh'
+    elif lang[-2:] == 'SA':
+        region_code = 'xa-' + lang.split('-')[0]
+    elif lang[-2:] == 'GB':
+        region_code = 'uk-' + lang.split('-')[0]
     else:
-        locale = params['language'].split('_')
-        if len(locale) == 2:
+        region_code = lang.split('-')
+        if len(region_code) == 2:
             # country code goes first
-            locale = locale[1].lower() + '-' + locale[0].lower()
+            region_code = region_code[1].lower() + '-' + region_code[0].lower()
         else:
             # tries to get a country code from language
-            locale = locale[0].lower()
-            lang_codes = [x[0] for x in language_codes]
-            for lc in lang_codes:
-                lc = lc.split('_')
-                if locale == lc[0]:
-                    locale = lc[1].lower() + '-' + lc[0].lower()
+            region_code = region_code[0].lower()
+            for lc in supported_languages:
+                lc = lc.split('-')
+                if region_code == lc[0]:
+                    region_code = lc[1].lower() + '-' + lc[0].lower()
                     break
+    return region_code
 
-    if locale:
+
+# do search-request
+def request(query, params):
+    if params['time_range'] and params['time_range'] not in time_range_dict:
+        return params
+
+    offset = (params['pageno'] - 1) * 30
+
+    region_code = get_region_code(params['language'])
+    if region_code:
         params['url'] = url.format(
-            query=urlencode({'q': query, 'kl': locale}), offset=offset)
+            query=urlencode({'q': query, 'kl': region_code}), offset=offset, dc_param=offset)
     else:
         params['url'] = url.format(
-            query=urlencode({'q': query}), offset=offset)
+            query=urlencode({'q': query}), offset=offset, dc_param=offset)
 
     if params['time_range'] in time_range_dict:
         params['url'] += time_range_url.format(range=time_range_dict[params['time_range']])
@@ -99,3 +121,17 @@ def response(resp):
 
     # return results
     return results
+
+
+# get supported languages from their site
+def _fetch_supported_languages(resp):
+
+    # response is a js file with regions as an embedded object
+    response_page = resp.text
+    response_page = response_page[response_page.find('regions:{') + 8:]
+    response_page = response_page[:response_page.find('}') + 1]
+
+    regions_json = loads(response_page)
+    supported_languages = map((lambda x: x[3:] + '-' + x[:2].upper()), regions_json.keys())
+
+    return supported_languages

searx/engines/duckduckgo_definitions.py

@@ -1,9 +1,10 @@
 import json
-from urllib import urlencode
-from re import compile, sub
 from lxml import html
-from searx.utils import html_to_text
+from re import compile
 from searx.engines.xpath import extract_text
+from searx.engines.duckduckgo import _fetch_supported_languages, supported_languages_url
+from searx.url_utils import urlencode
+from searx.utils import html_to_text
 
 url = 'https://api.duckduckgo.com/'\
     + '?{query}&format=json&pretty=0&no_redirect=1&d=1'
@@ -23,7 +24,7 @@ def result_to_text(url, text, htmlResult):
 
 def request(query, params):
     params['url'] = url.format(query=urlencode({'q': query}))
-    params['headers']['Accept-Language'] = params['language']
+    params['headers']['Accept-Language'] = params['language'].split('-')[0]
     return params
 
 

searx/engines/duckduckgo_images.py

@@ -0,0 +1,91 @@
+"""
+ DuckDuckGo (Images)
+
+ @website     https://duckduckgo.com/
+ @provide-api yes (https://duckduckgo.com/api),
+              but images are not supported
+
+ @using-api   no
+ @results     JSON (site requires js to get images)
+ @stable      no (JSON can change)
+ @parse       url, title, img_src
+
+ @todo        avoid extra request
+"""
+
+from requests import get
+from json import loads
+from searx.engines.xpath import extract_text
+from searx.engines.duckduckgo import _fetch_supported_languages, supported_languages_url, get_region_code
+from searx.url_utils import urlencode
+
+# engine dependent config
+categories = ['images']
+paging = True
+language_support = True
+safesearch = True
+
+# search-url
+images_url = 'https://duckduckgo.com/i.js?{query}&s={offset}&p={safesearch}&o=json&vqd={vqd}'
+site_url = 'https://duckduckgo.com/?{query}&iar=images&iax=1&ia=images'
+
+
+# run query in site to get vqd number needed for requesting images
+# TODO: find a way to get this number without an extra request (is it a hash of the query?)
+def get_vqd(query):
+    res = get(site_url.format(query=urlencode({'q': query})))
+    content = res.text
+    vqd = content[content.find('vqd=\'') + 5:]
+    vqd = vqd[:vqd.find('\'')]
+    return vqd
+
+
+# do search-request
+def request(query, params):
+    # to avoid running actual external requests when testing
+    if 'is_test' not in params:
+        vqd = get_vqd(query)
+    else:
+        vqd = '12345'
+
+    offset = (params['pageno'] - 1) * 50
+
+    safesearch = params['safesearch'] - 1
+
+    region_code = get_region_code(params['language'])
+    if region_code:
+        params['url'] = images_url.format(
+            query=urlencode({'q': query, 'l': region_code}), offset=offset, safesearch=safesearch, vqd=vqd)
+    else:
+        params['url'] = images_url.format(
+            query=urlencode({'q': query}), offset=offset, safesearch=safesearch, vqd=vqd)
+
+    return params
+
+
+# get response from search-request
+def response(resp):
+    results = []
+
+    content = resp.text
+    try:
+        res_json = loads(content)
+    except:
+        return []
+
+    # parse results
+    for result in res_json['results']:
+        title = result['title']
+        url = result['url']
+        thumbnail = result['thumbnail']
+        image = result['image']
+
+        # append result
+        results.append({'template': 'images.html',
+                        'title': title,
+                        'content': '',
+                        'thumbnail_src': thumbnail,
+                        'img_src': image,
+                        'url': url})
+
+    return results

searx/engines/faroo.py

@@ -10,10 +10,10 @@
  @parse       url, title, content, publishedDate, img_src
 """
 
-from urllib import urlencode
 from json import loads
 import datetime
 from searx.utils import searx_useragent
+from searx.url_utils import urlencode
 
 # engine dependent config
 categories = ['general', 'news']

searx/engines/fdroid.py

@@ -9,10 +9,9 @@
  @parse        url, title, content
 """
 
-from cgi import escape
-from urllib import urlencode
-from searx.engines.xpath import extract_text
 from lxml import html
+from searx.engines.xpath import extract_text
+from searx.url_utils import urlencode
 
 # engine dependent config
 categories = ['files']
@@ -25,8 +24,7 @@ search_url = base_url + 'repository/browse/?{query}'
 
 # do search-request
 def request(query, params):
-    query = urlencode({'fdfilter': query,
-                       'fdpage': params['pageno']})
+    query = urlencode({'fdfilter': query, 'fdpage': params['pageno']})
     params['url'] = search_url.format(query=query)
     return params
 
@@ -43,7 +41,7 @@ def response(resp):
         img_src = app.xpath('.//img/@src')[0]
 
         content = extract_text(app.xpath('./p')[0])
-        content = escape(content.replace(title, '', 1).strip())
+        content = content.replace(title, '', 1).strip()
 
         results.append({'url': url,
                         'title': title,

searx/engines/filecrop.py

@@ -1,5 +1,9 @@
-from urllib import urlencode
-from HTMLParser import HTMLParser
+from searx.url_utils import urlencode
+
+try:
+    from HTMLParser import HTMLParser
+except:
+    from html.parser import HTMLParser
 
 url = 'http://www.filecrop.com/'
 search_url = url + '/search.php?{query}&size_i=0&size_f=100000000&engine_r=1&engine_d=1&engine_e=1&engine_4=1&engine_m=1&pos={index}'  # noqa
@@ -73,8 +77,7 @@ class FilecropResultParser(HTMLParser):
 
 def request(query, params):
     index = 1 + (params['pageno'] - 1) * 30
-    params['url'] = search_url.format(query=urlencode({'w': query}),
-                                      index=index)
+    params['url'] = search_url.format(query=urlencode({'w': query}), index=index)
     return params
 
 

searx/engines/flickr.py

@@ -13,8 +13,8 @@
  More info on api-key : https://www.flickr.com/services/apps/create/
 """
 
-from urllib import urlencode
 from json import loads
+from searx.url_utils import urlencode
 
 categories = ['images']
 
@@ -77,21 +77,13 @@ def response(resp):
 
         url = build_flickr_url(photo['owner'], photo['id'])
 
-        title = photo['title']
-
-        content = '<span class="photo-author">' +\
-                  photo['ownername'] +\
-                  '</span><br />' +\
-                  '<span class="description">' +\
-                  photo['description']['_content'] +\
-                  '</span>'
-
         # append result
         results.append({'url': url,
-                        'title': title,
+                        'title': photo['title'],
                         'img_src': img_src,
                         'thumbnail_src': thumbnail_src,
-                        'content': content,
+                        'content': photo['description']['_content'],
+                        'author': photo['ownername'],
                         'template': 'images.html'})
 
     # return results

searx/engines/flickr_noapi.py

@@ -12,10 +12,11 @@
  @parse       url, title, thumbnail, img_src
 """
 
-from urllib import urlencode
 from json import loads
+from time import time
 import re
 from searx.engines import logger
+from searx.url_utils import urlencode
 
 
 logger = logger.getChild('flickr-noapi')
@@ -24,21 +25,32 @@ categories = ['images']
 
 url = 'https://www.flickr.com/'
 search_url = url + 'search?{query}&page={page}'
+time_range_url = '&min_upload_date={start}&max_upload_date={end}'
 photo_url = 'https://www.flickr.com/photos/{userid}/{photoid}'
 regex = re.compile(r"\"search-photos-lite-models\",\"photos\":(.*}),\"totalItems\":", re.DOTALL)
 image_sizes = ('o', 'k', 'h', 'b', 'c', 'z', 'n', 'm', 't', 'q', 's')
 
 paging = True
+time_range_support = True
+time_range_dict = {'day': 60 * 60 * 24,
+                   'week': 60 * 60 * 24 * 7,
+                   'month': 60 * 60 * 24 * 7 * 4,
+                   'year': 60 * 60 * 24 * 7 * 52}
 
 
 def build_flickr_url(user_id, photo_id):
     return photo_url.format(userid=user_id, photoid=photo_id)
 
 
-def request(query, params):
-    params['url'] = search_url.format(query=urlencode({'text': query}),
-                                      page=params['pageno'])
+def _get_time_range_url(time_range):
+    if time_range in time_range_dict:
+        return time_range_url.format(start=time(), end=str(int(time()) - time_range_dict[time_range]))
+    return ''
 
+
+def request(query, params):
+    params['url'] = (search_url.format(query=urlencode({'text': query}), page=params['pageno'])
+                     + _get_time_range_url(params['time_range']))
     return params
 
 
@@ -91,16 +103,15 @@ def response(resp):
 
         title = photo.get('title', '')
 
-        content = '<span class="photo-author">' +\
-                  photo['username'] +\
-                  '</span><br />'
+        author = photo['username']
 
         # append result
         results.append({'url': url,
                         'title': title,
                         'img_src': img_src,
                         'thumbnail_src': thumbnail_src,
-                        'content': content,
+                        'content': '',
+                        'author': author,
                         'template': 'images.html'})
 
     return results

searx/engines/framalibre.py

@@ -0,0 +1,69 @@
+"""
+ FramaLibre (It)
+
+ @website     https://framalibre.org/
+ @provide-api no
+
+ @using-api   no
+ @results     HTML
+ @stable      no (HTML can change)
+ @parse       url, title, content, thumbnail, img_src
+"""
+
+from cgi import escape
+from lxml import html
+from searx.engines.xpath import extract_text
+from searx.url_utils import urljoin, urlencode
+
+# engine dependent config
+categories = ['it']
+paging = True
+
+# search-url
+base_url = 'https://framalibre.org/'
+search_url = base_url + 'recherche-par-crit-res?{query}&page={offset}'
+
+# specific xpath variables
+results_xpath = '//div[@class="nodes-list-row"]/div[contains(@typeof,"sioc:Item")]'
+link_xpath = './/h3[@class="node-title"]/a[@href]'
+thumbnail_xpath = './/img[@class="media-object img-responsive"]/@src'
+content_xpath = './/div[@class="content"]//p'
+
+
+# do search-request
+def request(query, params):
+    offset = (params['pageno'] - 1)
+    params['url'] = search_url.format(query=urlencode({'keys': query}),
+                                      offset=offset)
+
+    return params
+
+
+# get response from search-request
+def response(resp):
+    results = []
+
+    dom = html.fromstring(resp.text)
+
+    # parse results
+    for result in dom.xpath(results_xpath):
+        link = result.xpath(link_xpath)[0]
+        href = urljoin(base_url, link.attrib.get('href'))
+        # there's also a span (class="rdf-meta element-hidden" property="dc:title")'s content property for this...
+        title = escape(extract_text(link))
+        thumbnail_tags = result.xpath(thumbnail_xpath)
+        thumbnail = None
+        if len(thumbnail_tags) > 0:
+            thumbnail = extract_text(thumbnail_tags[0])
+            if thumbnail[0] == '/':
+                thumbnail = base_url + thumbnail
+        content = escape(extract_text(result.xpath(content_xpath)))
+
+        # append result
+        results.append({'url': href,
+                        'title': title,
+                        'img_src': thumbnail,
+                        'content': content})
+
+    # return results
+    return results

searx/engines/frinkiac.py

@@ -10,7 +10,7 @@ Frinkiac (Images)
 """
 
 from json import loads
-from urllib import urlencode
+from searx.url_utils import urlencode
 
 categories = ['images']
 

searx/engines/gigablast.py

@@ -10,11 +10,10 @@
  @parse       url, title, content
 """
 
-from cgi import escape
 from json import loads
-from random import randint
 from time import time
-from urllib import urlencode
+from lxml.html import fromstring
+from searx.url_utils import urlencode
 
 # engine dependent config
 categories = ['general']
@@ -41,6 +40,8 @@ url_xpath = './/url'
 title_xpath = './/title'
 content_xpath = './/sum'
 
+supported_languages_url = 'https://gigablast.com/search?&rxikd=1'
+
 
 # do search-request
 def request(query, params):
@@ -49,7 +50,9 @@ def request(query, params):
     if params['language'] == 'all':
         language = 'xx'
     else:
-        language = params['language'][0:2]
+        language = params['language'].replace('-', '_').lower()
+        if language.split('-')[0] != 'zh':
+            language = language.split('-')[0]
 
     if params['safesearch'] >= 1:
         safesearch = 1
@@ -78,8 +81,26 @@ def response(resp):
     for result in response_json['results']:
         # append result
         results.append({'url': result['url'],
-                        'title': escape(result['title']),
-                        'content': escape(result['sum'])})
+                        'title': result['title'],
+                        'content': result['sum']})
 
     # return results
     return results
+
+
+# get supported languages from their site
+def _fetch_supported_languages(resp):
+    supported_languages = []
+    dom = fromstring(resp.text)
+    links = dom.xpath('//span[@id="menu2"]/a')
+    for link in links:
+        href = link.xpath('./@href')[0].split('lang%3A')
+        if len(href) == 2:
+            code = href[1].split('_')
+            if len(code) == 2:
+                code = code[0] + '-' + code[1].upper()
+            else:
+                code = code[0]
+            supported_languages.append(code)
+
+    return supported_languages

searx/engines/github.py

@@ -10,9 +10,8 @@
  @parse       url, title, content
 """
 
-from urllib import urlencode
 from json import loads
-from cgi import escape
+from searx.url_utils import urlencode
 
 # engine dependent config
 categories = ['it']
@@ -48,7 +47,7 @@ def response(resp):
         url = res['html_url']
 
         if res['description']:
-            content = escape(res['description'][:500])
+            content = res['description'][:500]
         else:
             content = ''
 

searx/engines/google.py

@@ -9,12 +9,10 @@
 # @parse       url, title, content, suggestion
 
 import re
-from cgi import escape
-from urllib import urlencode
-from urlparse import urlparse, parse_qsl
 from lxml import html, etree
 from searx.engines.xpath import extract_text, extract_url
-from searx.search import logger
+from searx import logger
+from searx.url_utils import urlencode, urlparse, parse_qsl
 
 logger = logger.getChild('google engine')
 
@@ -96,13 +94,15 @@ search_url = ('https://{hostname}' +
 time_range_search = "&tbs=qdr:{range}"
 time_range_dict = {'day': 'd',
                    'week': 'w',
-                   'month': 'm'}
+                   'month': 'm',
+                   'year': 'y'}
 
 # other URLs
 map_hostname_start = 'maps.google.'
 maps_path = '/maps'
 redirect_path = '/url'
 images_path = '/images'
+supported_languages_url = 'https://www.google.com/preferences?#languages'
 
 # specific xpath variables
 results_xpath = '//div[@class="g"]'
@@ -111,6 +111,7 @@ title_xpath = './/h3'
 content_xpath = './/span[@class="st"]'
 content_misc_xpath = './/div[@class="f slp"]'
 suggestion_xpath = '//p[@class="_Bmc"]'
+spelling_suggestion_xpath = '//a[@class="spell"]'
 
 # map : detail location
 map_address_xpath = './/div[@class="s"]//table//td[2]/span/text()'
@@ -155,7 +156,7 @@ def parse_url(url_string, google_hostname):
 def extract_text_from_dom(result, xpath):
     r = result.xpath(xpath)
     if len(r) > 0:
-        return escape(extract_text(r[0]))
+        return extract_text(r[0])
     return None
 
 
@@ -167,8 +168,12 @@ def request(query, params):
         language = 'en'
         country = 'US'
         url_lang = ''
+    elif params['language'][:2] == 'jv':
+        language = 'jw'
+        country = 'ID'
+        url_lang = 'lang_jw'
     else:
-        language_array = params['language'].lower().split('_')
+        language_array = params['language'].lower().split('-')
         if len(language_array) == 2:
             country = language_array[1]
         else:
@@ -212,6 +217,16 @@ def response(resp):
     # convert the text to dom
     dom = html.fromstring(resp.text)
 
+    instant_answer = dom.xpath('//div[@id="_vBb"]//text()')
+    if instant_answer:
+        results.append({'answer': u' '.join(instant_answer)})
+    try:
+        results_num = int(dom.xpath('//div[@id="resultStats"]//text()')[0]
+                          .split()[1].replace(',', ''))
+        results.append({'number_of_results': results_num})
+    except:
+        pass
+
     # parse results
     for result in dom.xpath(results_xpath):
         try:
@@ -264,7 +279,10 @@ def response(resp):
     # parse suggestion
     for suggestion in dom.xpath(suggestion_xpath):
         # append suggestion
-        results.append({'suggestion': escape(extract_text(suggestion))})
+        results.append({'suggestion': extract_text(suggestion)})
+
+    for correction in dom.xpath(spelling_suggestion_xpath):
+        results.append({'correction': extract_text(correction)})
 
     # return results
     return results
@@ -355,3 +373,16 @@ def attributes_to_html(attributes):
         retval = retval + '<tr><th>' + a.get('label') + '</th><td>' + value + '</td></tr>'
     retval = retval + '</table>'
     return retval
+
+
+# get supported languages from their site
+def _fetch_supported_languages(resp):
+    supported_languages = {}
+    dom = html.fromstring(resp.text)
+    options = dom.xpath('//table//td/font/label/span')
+    for option in options:
+        code = option.xpath('./@id')[0][1:]
+        name = option.text.title()
+        supported_languages[code] = {"name": name}
+
+    return supported_languages

searx/engines/google_images.py

@@ -10,9 +10,11 @@
  @parse       url, title, img_src
 """
 
-from urllib import urlencode
+from datetime import date, timedelta
 from json import loads
 from lxml import html
+from searx.url_utils import urlencode
+
 
 # engine dependent config
 categories = ['images']
@@ -29,6 +31,7 @@ search_url = 'https://www.google.com/search'\
     '&yv=2'\
     '&{search_options}'
 time_range_attr = "qdr:{range}"
+time_range_custom_attr = "cdr:1,cd_min:{start},cd_max{end}"
 time_range_dict = {'day': 'd',
                    'week': 'w',
                    'month': 'm'}
@@ -36,7 +39,6 @@ time_range_dict = {'day': 'd',
 
 # do search-request
 def request(query, params):
-
     search_options = {
         'ijn': params['pageno'] - 1,
         'start': (params['pageno'] - 1) * number_of_results
@@ -44,6 +46,12 @@ def request(query, params):
 
     if params['time_range'] in time_range_dict:
         search_options['tbs'] = time_range_attr.format(range=time_range_dict[params['time_range']])
+    elif params['time_range'] == 'year':
+        now = date.today()
+        then = now - timedelta(days=365)
+        start = then.strftime('%m/%d/%Y')
+        end = now.strftime('%m/%d/%Y')
+        search_options['tbs'] = time_range_custom_attr.format(start=start, end=end)
 
     if safesearch and params['safesearch']:
         search_options['safe'] = 'on'

searx/engines/google_news.py

@@ -1,41 +1,58 @@
 """
  Google (News)
 
- @website     https://www.google.com
- @provide-api yes (https://developers.google.com/web-search/docs/),
-              deprecated!
+ @website     https://news.google.com
+ @provide-api no
 
- @using-api   yes
- @results     JSON
- @stable      yes (but deprecated)
+ @using-api   no
+ @results     HTML
+ @stable      no
  @parse       url, title, content, publishedDate
 """
 
-from urllib import urlencode
-from json import loads
-from dateutil import parser
+from lxml import html
+from searx.engines.google import _fetch_supported_languages, supported_languages_url
+from searx.url_utils import urlencode
 
 # search-url
 categories = ['news']
 paging = True
 language_support = True
+safesearch = True
+time_range_support = True
+number_of_results = 10
 
-# engine dependent config
-url = 'https://ajax.googleapis.com/'
-search_url = url + 'ajax/services/search/news?v=2.0&start={offset}&rsz=large&safe=off&filter=off&{query}&hl={lang}'
+search_url = 'https://www.google.com/search'\
+    '?{query}'\
+    '&tbm=nws'\
+    '&gws_rd=cr'\
+    '&{search_options}'
+time_range_attr = "qdr:{range}"
+time_range_dict = {'day': 'd',
+                   'week': 'w',
+                   'month': 'm',
+                   'year': 'y'}
 
 
 # do search-request
 def request(query, params):
-    offset = (params['pageno'] - 1) * 8
 
-    language = 'en-US'
+    search_options = {
+        'start': (params['pageno'] - 1) * number_of_results
+    }
+
+    if params['time_range'] in time_range_dict:
+        search_options['tbs'] = time_range_attr.format(range=time_range_dict[params['time_range']])
+
+    if safesearch and params['safesearch']:
+        search_options['safe'] = 'on'
+
+    params['url'] = search_url.format(query=urlencode({'q': query}),
+                                      search_options=urlencode(search_options))
+
     if params['language'] != 'all':
-        language = params['language'].replace('_', '-')
-
-    params['url'] = search_url.format(offset=offset,
-                                      query=urlencode({'q': query}),
-                                      lang=language)
+        language_array = params['language'].lower().split('-')
+        params['url'] += '&lr=lang_' + language_array[0]
 
     return params
 
@@ -44,24 +61,24 @@ def request(query, params):
 def response(resp):
     results = []
 
-    search_res = loads(resp.text)
-
-    # return empty array if there are no results
-    if not search_res.get('responseData', {}).get('results'):
-        return []
+    dom = html.fromstring(resp.text)
 
     # parse results
-    for result in search_res['responseData']['results']:
-        # parse publishedDate
-        publishedDate = parser.parse(result['publishedDate'])
-        if 'url' not in result:
+    for result in dom.xpath('//div[@class="g"]|//div[@class="g _cy"]'):
+        try:
+            r = {
+                'url': result.xpath('.//div[@class="_cnc"]//a/@href')[0],
+                'title': ''.join(result.xpath('.//div[@class="_cnc"]//h3//text()')),
+                'content': ''.join(result.xpath('.//div[@class="st"]//text()')),
+            }
+        except:
             continue
 
-        # append result
-        results.append({'url': result['unescapedUrl'],
-                        'title': result['titleNoFormatting'],
-                        'publishedDate': publishedDate,
-                        'content': result['content']})
+        imgs = result.xpath('.//img/@src')
+        if len(imgs) and not imgs[0].startswith('data'):
+            r['img_src'] = imgs[0]
+
+        results.append(r)
 
     # return results
     return results

searx/engines/ina.py

@@ -12,11 +12,15 @@
 # @todo        embedded (needs some md5 from video page)
 
 from json import loads
-from urllib import urlencode
 from lxml import html
-from HTMLParser import HTMLParser
-from searx.engines.xpath import extract_text
 from dateutil import parser
+from searx.engines.xpath import extract_text
+from searx.url_utils import urlencode
+
+try:
+    from HTMLParser import HTMLParser
+except:
+    from html.parser import HTMLParser
 
 # engine dependent config
 categories = ['videos']

searx/engines/json_engine.py

@@ -1,11 +1,16 @@
-from urllib import urlencode
-from json import loads
 from collections import Iterable
+from json import loads
+from sys import version_info
+from searx.url_utils import urlencode
+
+if version_info[0] == 3:
+    unicode = str
 
 search_url = None
 url_query = None
 content_query = None
 title_query = None
+paging = False
 suggestion_query = ''
 results_query = ''
 
@@ -20,7 +25,7 @@ first_page_num = 1
 
 def iterate(iterable):
     if type(iterable) == dict:
-        it = iterable.iteritems()
+        it = iterable.items()
 
     else:
         it = enumerate(iterable)

searx/engines/kickass.py

@@ -10,19 +10,18 @@
  @parse       url, title, content, seed, leech, magnetlink
 """
 
-from urlparse import urljoin
-from cgi import escape
-from urllib import quote
 from lxml import html
 from operator import itemgetter
 from searx.engines.xpath import extract_text
+from searx.utils import get_torrent_size, convert_str_to_int
+from searx.url_utils import quote, urljoin
 
 # engine dependent config
 categories = ['videos', 'music', 'files']
 paging = True
 
 # search-url
-url = 'https://kickass.to/'
+url = 'https://kickass.cd/'
 search_url = url + 'search/{search_term}/{pageno}/'
 
 # specific xpath variables
@@ -56,42 +55,17 @@ def response(resp):
         link = result.xpath('.//a[@class="cellMainLink"]')[0]
         href = urljoin(url, link.attrib['href'])
         title = extract_text(link)
-        content = escape(extract_text(result.xpath(content_xpath)))
-        seed = result.xpath('.//td[contains(@class, "green")]/text()')[0]
-        leech = result.xpath('.//td[contains(@class, "red")]/text()')[0]
-        filesize = result.xpath('.//td[contains(@class, "nobr")]/text()')[0]
-        filesize_multiplier = result.xpath('.//td[contains(@class, "nobr")]//span/text()')[0]
-        files = result.xpath('.//td[contains(@class, "center")][2]/text()')[0]
+        content = extract_text(result.xpath(content_xpath))
+        seed = extract_text(result.xpath('.//td[contains(@class, "green")]'))
+        leech = extract_text(result.xpath('.//td[contains(@class, "red")]'))
+        filesize_info = extract_text(result.xpath('.//td[contains(@class, "nobr")]'))
+        files = extract_text(result.xpath('.//td[contains(@class, "center")][2]'))
 
-        # convert seed to int if possible
-        if seed.isdigit():
-            seed = int(seed)
-        else:
-            seed = 0
+        seed = convert_str_to_int(seed)
+        leech = convert_str_to_int(leech)
 
-        # convert leech to int if possible
-        if leech.isdigit():
-            leech = int(leech)
-        else:
-            leech = 0
-
-        # convert filesize to byte if possible
-        try:
-            filesize = float(filesize)
-
-            # convert filesize to byte
-            if filesize_multiplier == 'TB':
-                filesize = int(filesize * 1024 * 1024 * 1024 * 1024)
-            elif filesize_multiplier == 'GB':
-                filesize = int(filesize * 1024 * 1024 * 1024)
-            elif filesize_multiplier == 'MB':
-                filesize = int(filesize * 1024 * 1024)
-            elif filesize_multiplier == 'KB':
-                filesize = int(filesize * 1024)
-        except:
-            filesize = None
-
-        # convert files to int if possible
+        filesize, filesize_multiplier = filesize_info.split()
+        filesize = get_torrent_size(filesize, filesize_multiplier)
         if files.isdigit():
             files = int(files)
         else:

searx/engines/mediawiki.py

@@ -14,13 +14,14 @@
 
 from json import loads
 from string import Formatter
-from urllib import urlencode, quote
+from searx.url_utils import urlencode, quote
 
 # engine dependent config
 categories = ['general']
 language_support = True
 paging = True
 number_of_results = 1
+search_type = 'nearmatch'  # possible values: title, text, nearmatch
 
 # search-url
 base_url = 'https://{language}.wikipedia.org/'
@@ -30,7 +31,7 @@ search_postfix = 'w/api.php?action=query'\
     '&format=json'\
     '&sroffset={offset}'\
     '&srlimit={limit}'\
-    '&srwhat=nearmatch'  # search for a near match in the title
+    '&srwhat={searchtype}'
 
 
 # do search-request
@@ -39,14 +40,15 @@ def request(query, params):
 
     string_args = dict(query=urlencode({'srsearch': query}),
                        offset=offset,
-                       limit=number_of_results)
+                       limit=number_of_results,
+                       searchtype=search_type)
 
     format_strings = list(Formatter().parse(base_url))
 
     if params['language'] == 'all':
         language = 'en'
     else:
-        language = params['language'].split('_')[0]
+        language = params['language'].split('-')[0]
 
     # format_string [('https://', 'language', '', None), ('.wikipedia.org/', None, None, None)]
     if any(x[1] == 'language' for x in format_strings):

searx/engines/mixcloud.py

@@ -11,8 +11,8 @@
 """
 
 from json import loads
-from urllib import urlencode
 from dateutil import parser
+from searx.url_utils import urlencode
 
 # engine dependent config
 categories = ['music']

searx/engines/nyaa.py

@@ -9,10 +9,9 @@
  @parse        url, title, content, seed, leech, torrentfile
 """
 
-from cgi import escape
-from urllib import urlencode
 from lxml import html
 from searx.engines.xpath import extract_text
+from searx.url_utils import urlencode
 
 # engine dependent config
 categories = ['files', 'images', 'videos', 'music']
@@ -78,7 +77,7 @@ def response(resp):
 
         # torrent title
         page_a = result.xpath(xpath_title)[0]
-        title = escape(extract_text(page_a))
+        title = extract_text(page_a)
 
         # link to the page
         href = page_a.attrib.get('href')
@@ -90,7 +89,7 @@ def response(resp):
         try:
             file_size, suffix = result.xpath(xpath_filesize)[0].split(' ')
             file_size = int(float(file_size) * get_filesize_mul(suffix))
-        except Exception as e:
+        except:
             file_size = None
 
         # seed count
@@ -105,7 +104,6 @@ def response(resp):
         # content string contains all information not included into template
         content = 'Category: "{category}". Downloaded {downloads} times.'
         content = content.format(category=category, downloads=downloads)
-        content = escape(content)
 
         results.append({'url': href,
                         'title': title,

searx/engines/openstreetmap.py

@@ -11,7 +11,6 @@
 """
 
 from json import loads
-from searx.utils import searx_useragent
 
 # engine dependent config
 categories = ['map']
@@ -27,9 +26,6 @@ result_base_url = 'https://openstreetmap.org/{osm_type}/{osm_id}'
 def request(query, params):
     params['url'] = base_url + search_string.format(query=query)
 
-    # using searx User-Agent
-    params['headers']['User-Agent'] = searx_useragent()
-
     return params
 
 
@@ -43,7 +39,7 @@ def response(resp):
         if 'display_name' not in r:
             continue
 
-        title = r['display_name']
+        title = r['display_name'] or u''
         osm_type = r.get('osm_type', r.get('type'))
         url = result_base_url.format(osm_type=osm_type,
                                      osm_id=r['osm_id'])

searx/engines/pdbe.py

@@ -0,0 +1,109 @@
+"""
+ PDBe (Protein Data Bank in Europe)
+
+ @website       https://www.ebi.ac.uk/pdbe
+ @provide-api   yes (https://www.ebi.ac.uk/pdbe/api/doc/search.html),
+                unlimited
+ @using-api     yes
+ @results       python dictionary (from json)
+ @stable        yes
+ @parse         url, title, content, img_src
+"""
+
+from json import loads
+from flask_babel import gettext
+
+categories = ['science']
+
+hide_obsolete = False
+
+# status codes of unpublished entries
+pdb_unpublished_codes = ['HPUB', 'HOLD', 'PROC', 'WAIT', 'AUTH', 'AUCO', 'REPL', 'POLC', 'REFI', 'TRSF', 'WDRN']
+# url for api query
+pdbe_solr_url = 'https://www.ebi.ac.uk/pdbe/search/pdb/select?'
+# base url for results
+pdbe_entry_url = 'https://www.ebi.ac.uk/pdbe/entry/pdb/{pdb_id}'
+# link to preview image of structure
+pdbe_preview_url = 'https://www.ebi.ac.uk/pdbe/static/entry/{pdb_id}_deposited_chain_front_image-200x200.png'
+
+
+def request(query, params):
+
+    params['url'] = pdbe_solr_url
+    params['method'] = 'POST'
+    params['data'] = {
+        'q': query,
+        'wt': "json"  # request response in parsable format
+    }
+    return params
+
+
+def construct_body(result):
+    # set title
+    title = result['title']
+
+    # construct content body
+    content = """{title}<br />{authors} {journal} <strong>{volume}</strong>&nbsp;{page} ({year})"""
+
+    # replace placeholders with actual content
+    try:
+        if result['journal']:
+            content = content.format(
+                title=result['citation_title'],
+                authors=result['entry_author_list'][0], journal=result['journal'], volume=result['journal_volume'],
+                page=result['journal_page'], year=result['citation_year'])
+        else:
+            content = content.format(
+                title=result['citation_title'],
+                authors=result['entry_author_list'][0], journal='', volume='', page='', year=result['release_year'])
+        img_src = pdbe_preview_url.format(pdb_id=result['pdb_id'])
+    except (KeyError):
+        content = None
+        img_src = None
+
+    # construct url for preview image
+    try:
+        img_src = pdbe_preview_url.format(pdb_id=result['pdb_id'])
+    except (KeyError):
+        img_src = None
+
+    return [title, content, img_src]
+
+
+def response(resp):
+
+    results = []
+    json = loads(resp.text)['response']['docs']
+
+    # parse results
+    for result in json:
+        # catch obsolete entries and mark them accordingly
+        if result['status'] in pdb_unpublished_codes:
+            continue
+        if hide_obsolete:
+            continue
+        if result['status'] == 'OBS':
+            # expand title to add some sort of warning message
+            title = gettext('{title}&nbsp;(OBSOLETE)').format(title=result['title'])
+            superseded_url = pdbe_entry_url.format(pdb_id=result['superseded_by'])
+
+            # since we can't construct a proper body from the response, we'll make up our own
+            msg_superseded = gettext("This entry has been superseded by")
+            content = '<em>{msg_superseded} \<a href="{url}">{pdb_id}</a></em>'.format(
+                msg_superseded=msg_superseded,
+                url=superseded_url,
+                pdb_id=result['superseded_by'], )
+
+            # obsoleted entries don't have preview images
+            img_src = None
+        else:
+            title, content, img_src = construct_body(result)
+
+        results.append({
+            'url': pdbe_entry_url.format(pdb_id=result['pdb_id']),
+            'title': title,
+            'content': content,
+            'img_src': img_src
+        })
+
+    return results

searx/engines/photon.py

@@ -10,9 +10,9 @@
  @parse       url, title
 """
 
-from urllib import urlencode
 from json import loads
 from searx.utils import searx_useragent
+from searx.url_utils import urlencode
 
 # engine dependent config
 categories = ['map']
@@ -26,7 +26,7 @@ search_string = 'api/?{query}&limit={limit}'
 result_base_url = 'https://openstreetmap.org/{osm_type}/{osm_id}'
 
 # list of supported languages
-allowed_languages = ['de', 'en', 'fr', 'it']
+supported_languages = ['de', 'en', 'fr', 'it']
 
 
 # do search-request
@@ -37,7 +37,7 @@ def request(query, params):
 
     if params['language'] != 'all':
         language = params['language'].split('_')[0]
-        if language in allowed_languages:
+        if language in supported_languages:
             params['url'] = params['url'] + "&lang=" + language
 
     # using searx User-Agent

searx/engines/piratebay.py

@@ -8,12 +8,10 @@
 # @stable      yes (HTML can change)
 # @parse       url, title, content, seed, leech, magnetlink
 
-from urlparse import urljoin
-from cgi import escape
-from urllib import quote
 from lxml import html
 from operator import itemgetter
 from searx.engines.xpath import extract_text
+from searx.url_utils import quote, urljoin
 
 # engine dependent config
 categories = ['videos', 'music', 'files']
@@ -62,7 +60,7 @@ def response(resp):
         link = result.xpath('.//div[@class="detName"]//a')[0]
         href = urljoin(url, link.attrib.get('href'))
         title = extract_text(link)
-        content = escape(extract_text(result.xpath(content_xpath)))
+        content = extract_text(result.xpath(content_xpath))
         seed, leech = result.xpath('.//td[@align="right"]/text()')[:2]
 
         # convert seed to int if possible

searx/engines/qwant.py

@@ -10,14 +10,16 @@
  @parse       url, title, content
 """
 
-from urllib import urlencode
-from json import loads
 from datetime import datetime
+from json import loads
+from searx.utils import html_to_text
+from searx.url_utils import urlencode
 
 # engine dependent config
 categories = None
 paging = True
 language_support = True
+supported_languages_url = 'https://qwant.com/region'
 
 category_to_keyword = {'general': 'web',
                        'images': 'images',
@@ -44,7 +46,16 @@ def request(query, params):
 
     # add language tag if specified
     if params['language'] != 'all':
-        params['url'] += '&locale=' + params['language'].lower()
+        if params['language'] == 'no' or params['language'].startswith('no-'):
+            params['language'] = params['language'].replace('no', 'nb', 1)
+        if params['language'].find('-') < 0:
+            # tries to get a country code from language
+            for lang in supported_languages:
+                lc = lang.split('-')
+                if params['language'] == lc[0]:
+                    params['language'] = lang
+                    break
+        params['url'] += '&locale=' + params['language'].replace('-', '_').lower()
 
     return params
 
@@ -66,9 +77,9 @@ def response(resp):
     # parse results
     for result in res.get('items', {}):
 
-        title = result['title']
+        title = html_to_text(result['title'])
         res_url = result['url']
-        content = result['desc']
+        content = html_to_text(result['desc'])
 
         if category_to_keyword.get(categories[0], '') == 'web':
             results.append({'title': title,
@@ -85,14 +96,45 @@ def response(resp):
                             'thumbnail_src': thumbnail_src,
                             'img_src': img_src})
 
-        elif (category_to_keyword.get(categories[0], '') == 'news' or
-              category_to_keyword.get(categories[0], '') == 'social'):
+        elif category_to_keyword.get(categories[0], '') == 'social':
             published_date = datetime.fromtimestamp(result['date'], None)
-
+            img_src = result.get('img', None)
             results.append({'url': res_url,
                             'title': title,
                             'publishedDate': published_date,
-                            'content': content})
+                            'content': content,
+                            'img_src': img_src})
+
+        elif category_to_keyword.get(categories[0], '') == 'news':
+            published_date = datetime.fromtimestamp(result['date'], None)
+            media = result.get('media', [])
+            if len(media) > 0:
+                img_src = media[0].get('pict', {}).get('url', None)
+            else:
+                img_src = None
+            results.append({'url': res_url,
+                            'title': title,
+                            'publishedDate': published_date,
+                            'content': content,
+                            'img_src': img_src})
 
-    # return results
     return results
+
+
+# get supported languages from their site
+def _fetch_supported_languages(resp):
+    # list of regions is embedded in page as a js object
+    response_text = resp.text
+    response_text = response_text[response_text.find('regionalisation'):]
+    response_text = response_text[response_text.find('{'):response_text.find(');')]
+
+    regions_json = loads(response_text)
+
+    supported_languages = []
+    for lang in regions_json['languages'].values():
+        if lang['code'] == 'nb':
+            lang['code'] = 'no'
+        for country in lang['countries']:
+            supported_languages.append(lang['code'] + '-' + country)
+
+    return supported_languages

searx/engines/reddit.py

@@ -11,10 +11,8 @@
 """
 
 import json
-from cgi import escape
-from urllib import urlencode
-from urlparse import urlparse, urljoin
 from datetime import datetime
+from searx.url_utils import urlencode, urljoin, urlparse
 
 # engine dependent config
 categories = ['general', 'images', 'news', 'social media']
@@ -27,8 +25,7 @@ search_url = base_url + 'search.json?{query}'
 
 # do search-request
 def request(query, params):
-    query = urlencode({'q': query,
-                       'limit': page_size})
+    query = urlencode({'q': query, 'limit': page_size})
     params['url'] = search_url.format(query=query)
 
     return params
@@ -68,7 +65,7 @@ def response(resp):
             img_results.append(params)
         else:
             created = datetime.fromtimestamp(data['created_utc'])
-            content = escape(data['selftext'])
+            content = data['selftext']
             if len(content) > 500:
                 content = content[:500] + '...'
             params['content'] = content

searx/engines/scanr_structures.py

@@ -10,9 +10,7 @@
  @parse       url, title, content, img_src
 """
 
-from urllib import urlencode
 from json import loads, dumps
-from dateutil import parser
 from searx.utils import html_to_text
 
 # engine dependent config
@@ -48,7 +46,7 @@ def response(resp):
     search_res = loads(resp.text)
 
     # return empty array if there are no results
-    if search_res.get('total') < 1:
+    if search_res.get('total', 0) < 1:
         return []
 
     # parse results

searx/engines/searchcode_code.py

@@ -10,8 +10,8 @@
  @parse       url, title, content
 """
 
-from urllib import urlencode
 from json import loads
+from searx.url_utils import urlencode
 
 
 # engine dependent config
@@ -31,13 +31,7 @@ code_endings = {'cs': 'c#',
 
 # do search-request
 def request(query, params):
-    params['url'] = search_url.format(query=urlencode({'q': query}),
-                                      pageno=params['pageno'] - 1)
-
-    # Disable SSL verification
-    # error: (60) SSL certificate problem: unable to get local issuer
-    # certificate
-    params['verify'] = False
+    params['url'] = search_url.format(query=urlencode({'q': query}), pageno=params['pageno'] - 1)
 
     return params
 

searx/engines/searchcode_doc.py

@@ -10,8 +10,8 @@
  @parse       url, title, content
 """
 
-from urllib import urlencode
 from json import loads
+from searx.url_utils import urlencode
 
 # engine dependent config
 categories = ['it']
@@ -24,13 +24,7 @@ search_url = url + 'api/search_IV/?{query}&p={pageno}'
 
 # do search-request
 def request(query, params):
-    params['url'] = search_url.format(query=urlencode({'q': query}),
-                                      pageno=params['pageno'] - 1)
-
-    # Disable SSL verification
-    # error: (60) SSL certificate problem: unable to get local issuer
-    # certificate
-    params['verify'] = False
+    params['url'] = search_url.format(query=urlencode({'q': query}), pageno=params['pageno'] - 1)
 
     return params
 
@@ -44,20 +38,12 @@ def response(resp):
     # parse results
     for result in search_results.get('results', []):
         href = result['url']
-        title = "[" + result['type'] + "] " +\
-                result['namespace'] +\
-                " " + result['name']
-        content = '<span class="highlight">[' +\
-                  result['type'] + "] " +\
-                  result['name'] + " " +\
-                  result['synopsis'] +\
-                  "</span><br />" +\
-                  result['description']
+        title = "[{}] {} {}".format(result['type'], result['namespace'], result['name'])
 
         # append result
         results.append({'url': href,
                         'title': title,
-                        'content': content})
+                        'content': result['description']})
 
     # return results
     return results

searx/engines/searx_engine.py

@@ -0,0 +1,57 @@
+"""
+ Searx (all)
+
+ @website     https://github.com/asciimoo/searx
+ @provide-api yes (https://asciimoo.ithub.io/searx/dev/search_api.html)
+
+ @using-api   yes
+ @results     JSON
+ @stable      yes (using api)
+ @parse       url, title, content
+"""
+
+from json import loads
+from searx.engines import categories as searx_categories
+
+
+categories = searx_categories.keys()
+
+# search-url
+instance_urls = []
+instance_index = 0
+
+
+# do search-request
+def request(query, params):
+    global instance_index
+    params['url'] = instance_urls[instance_index % len(instance_urls)]
+    params['method'] = 'POST'
+
+    instance_index += 1
+
+    params['data'] = {
+        'q': query,
+        'pageno': params['pageno'],
+        'language': params['language'],
+        'time_range': params['time_range'],
+        'category': params['category'],
+        'format': 'json'
+    }
+
+    return params
+
+
+# get response from search-request
+def response(resp):
+
+    response_json = loads(resp.text)
+    results = response_json['results']
+
+    for i in ('answers', 'infoboxes'):
+        results.extend(response_json[i])
+
+    results.extend({'suggestion': s} for s in response_json['suggestions'])
+
+    results.append({'number_of_results': response_json['number_of_results']})
+
+    return results

searx/engines/seedpeer.py

@@ -0,0 +1,75 @@
+#  Seedpeer (Videos, Music, Files)
+#
+# @website     http://seedpeer.eu
+# @provide-api no (nothing found)
+#
+# @using-api   no
+# @results     HTML (using search portal)
+# @stable      yes (HTML can change)
+# @parse       url, title, content, seed, leech, magnetlink
+
+from lxml import html
+from operator import itemgetter
+from searx.url_utils import quote, urljoin
+
+
+url = 'http://www.seedpeer.eu/'
+search_url = url + 'search/{search_term}/7/{page_no}.html'
+# specific xpath variables
+torrent_xpath = '//*[@id="body"]/center/center/table[2]/tr/td/a'
+alternative_torrent_xpath = '//*[@id="body"]/center/center/table[1]/tr/td/a'
+title_xpath = '//*[@id="body"]/center/center/table[2]/tr/td/a/text()'
+alternative_title_xpath = '//*[@id="body"]/center/center/table/tr/td/a'
+seeds_xpath = '//*[@id="body"]/center/center/table[2]/tr/td[4]/font/text()'
+alternative_seeds_xpath = '//*[@id="body"]/center/center/table/tr/td[4]/font/text()'
+peers_xpath = '//*[@id="body"]/center/center/table[2]/tr/td[5]/font/text()'
+alternative_peers_xpath = '//*[@id="body"]/center/center/table/tr/td[5]/font/text()'
+age_xpath = '//*[@id="body"]/center/center/table[2]/tr/td[2]/text()'
+alternative_age_xpath = '//*[@id="body"]/center/center/table/tr/td[2]/text()'
+size_xpath = '//*[@id="body"]/center/center/table[2]/tr/td[3]/text()'
+alternative_size_xpath = '//*[@id="body"]/center/center/table/tr/td[3]/text()'
+
+
+# do search-request
+def request(query, params):
+    params['url'] = search_url.format(search_term=quote(query),
+                                      page_no=params['pageno'] - 1)
+    return params
+
+
+# get response from search-request
+def response(resp):
+    results = []
+    dom = html.fromstring(resp.text)
+    torrent_links = dom.xpath(torrent_xpath)
+    if len(torrent_links) > 0:
+        seeds = dom.xpath(seeds_xpath)
+        peers = dom.xpath(peers_xpath)
+        titles = dom.xpath(title_xpath)
+        sizes = dom.xpath(size_xpath)
+        ages = dom.xpath(age_xpath)
+    else:  # under ~5 results uses a different xpath
+        torrent_links = dom.xpath(alternative_torrent_xpath)
+        seeds = dom.xpath(alternative_seeds_xpath)
+        peers = dom.xpath(alternative_peers_xpath)
+        titles = dom.xpath(alternative_title_xpath)
+        sizes = dom.xpath(alternative_size_xpath)
+        ages = dom.xpath(alternative_age_xpath)
+    # return empty array if nothing is found
+    if not torrent_links:
+        return []
+
+    # parse results
+    for index, result in enumerate(torrent_links):
+        link = result.attrib.get('href')
+        href = urljoin(url, link)
+        results.append({'url': href,
+                        'title': titles[index].text_content(),
+                        'content': '{}, {}'.format(sizes[index], ages[index]),
+                        'seed': seeds[index],
+                        'leech': peers[index],
+
+                        'template': 'torrent.html'})
+
+    # return results sorted by seeder
+    return sorted(results, key=itemgetter('seed'), reverse=True)

searx/engines/soundcloud.py

@@ -11,13 +11,17 @@
 """
 
 import re
-from StringIO import StringIO
 from json import loads
-from lxml import etree
-from urllib import urlencode, quote_plus
+from lxml import html
 from dateutil import parser
 from searx import logger
 from searx.poolrequests import get as http_get
+from searx.url_utils import quote_plus, urlencode
+
+try:
+    from cStringIO import StringIO
+except:
+    from io import StringIO
 
 # engine dependent config
 categories = ['music']
@@ -36,14 +40,15 @@ embedded_url = '<iframe width="100%" height="166" ' +\
     'scrolling="no" frameborder="no" ' +\
     'data-src="https://w.soundcloud.com/player/?url={uri}"></iframe>'
 
+cid_re = re.compile(r'client_id:"([^"]*)"', re.I | re.U)
+
 
 def get_client_id():
     response = http_get("https://soundcloud.com")
-    rx_namespace = {"re": "http://exslt.org/regular-expressions"}
 
     if response.ok:
-        tree = etree.parse(StringIO(response.content), etree.HTMLParser())
-        script_tags = tree.xpath("//script[re:match(@src, '(.*app.*js)')]", namespaces=rx_namespace)
+        tree = html.fromstring(response.content)
+        script_tags = tree.xpath("//script[contains(@src, '/assets/app')]")
         app_js_urls = [script_tag.get('src') for script_tag in script_tags if script_tag is not None]
 
         # extracts valid app_js urls from soundcloud.com content
@@ -51,7 +56,7 @@ def get_client_id():
             # gets app_js and searches for the clientid
             response = http_get(app_js_url)
             if response.ok:
-                cids = re.search(r'client_id:"([^"]*)"', response.content, re.M | re.I)
+                cids = cid_re.search(response.text)
                 if cids is not None and len(cids.groups()):
                     return cids.groups()[0]
     logger.warning("Unable to fetch guest client_id from SoundCloud, check parser!")

searx/engines/spotify.py

@@ -11,7 +11,7 @@
 """
 
 from json import loads
-from urllib import urlencode
+from searx.url_utils import urlencode
 
 # engine dependent config
 categories = ['music']
@@ -29,8 +29,7 @@ embedded_url = '<iframe data-src="https://embed.spotify.com/?uri=spotify:track:{
 def request(query, params):
     offset = (params['pageno'] - 1) * 20
 
-    params['url'] = search_url.format(query=urlencode({'q': query}),
-                                      offset=offset)
+    params['url'] = search_url.format(query=urlencode({'q': query}), offset=offset)
 
     return params
 
@@ -46,10 +45,11 @@ def response(resp):
         if result['type'] == 'track':
             title = result['name']
             url = result['external_urls']['spotify']
-            content = result['artists'][0]['name'] +\
-                " &bull; " +\
-                result['album']['name'] +\
-                " &bull; " + result['name']
+            content = u'{} - {} - {}'.format(
+                result['artists'][0]['name'],
+                result['album']['name'],
+                result['name'])
+
             embedded = embedded_url.format(audioid=result['id'])
 
             # append result

searx/engines/stackoverflow.py

@@ -10,11 +10,9 @@
  @parse       url, title, content
 """
 
-from urlparse import urljoin
-from cgi import escape
-from urllib import urlencode
 from lxml import html
 from searx.engines.xpath import extract_text
+from searx.url_utils import urlencode, urljoin
 
 # engine dependent config
 categories = ['it']
@@ -32,8 +30,7 @@ content_xpath = './/div[@class="excerpt"]'
 
 # do search-request
 def request(query, params):
-    params['url'] = search_url.format(query=urlencode({'q': query}),
-                                      pageno=params['pageno'])
+    params['url'] = search_url.format(query=urlencode({'q': query}), pageno=params['pageno'])
 
     return params
 
@@ -48,8 +45,8 @@ def response(resp):
     for result in dom.xpath(results_xpath):
         link = result.xpath(link_xpath)[0]
         href = urljoin(url, link.attrib.get('href'))
-        title = escape(extract_text(link))
-        content = escape(extract_text(result.xpath(content_xpath)))
+        title = extract_text(link)
+        content = extract_text(result.xpath(content_xpath))
 
         # append result
         results.append({'url': href,

searx/engines/startpage.py

@@ -11,7 +11,6 @@
 # @todo        paging
 
 from lxml import html
-from cgi import escape
 from dateutil import parser
 from datetime import datetime, timedelta
 import re
@@ -48,7 +47,7 @@ def request(query, params):
 
     # set language if specified
     if params['language'] != 'all':
-        params['data']['with_language'] = ('lang_' + params['language'].split('_')[0])
+        params['data']['with_language'] = ('lang_' + params['language'].split('-')[0])
 
     return params
 
@@ -57,7 +56,7 @@ def request(query, params):
 def response(resp):
     results = []
 
-    dom = html.fromstring(resp.content)
+    dom = html.fromstring(resp.text)
 
     # parse results
     for result in dom.xpath(results_xpath):
@@ -79,10 +78,10 @@ def response(resp):
         if re.match(r"^http(s|)://(www\.)?ixquick\.com/do/search\?.*$", url):
             continue
 
-        title = escape(extract_text(link))
+        title = extract_text(link)
 
         if result.xpath('./p[@class="desc clk"]'):
-            content = escape(extract_text(result.xpath('./p[@class="desc clk"]')))
+            content = extract_text(result.xpath('./p[@class="desc clk"]'))
         else:
             content = ''
 

searx/engines/subtitleseeker.py

@@ -10,11 +10,10 @@
  @parse       url, title, content
 """
 
-from cgi import escape
-from urllib import quote_plus
 from lxml import html
 from searx.languages import language_codes
 from searx.engines.xpath import extract_text
+from searx.url_utils import quote_plus
 
 # engine dependent config
 categories = ['videos']
@@ -23,7 +22,7 @@ language = ""
 
 # search-url
 url = 'http://www.subtitleseeker.com/'
-search_url = url + 'search/TITLES/{query}&p={pageno}'
+search_url = url + 'search/TITLES/{query}?p={pageno}'
 
 # specific xpath variables
 results_xpath = '//div[@class="boxRows"]'
@@ -44,10 +43,16 @@ def response(resp):
 
     search_lang = ""
 
-    if resp.search_params['language'] != 'all':
-        search_lang = [lc[1]
+    # dirty fix for languages named differenly in their site
+    if resp.search_params['language'][:2] == 'fa':
+        search_lang = 'Farsi'
+    elif resp.search_params['language'] == 'pt-BR':
+        search_lang = 'Brazilian'
+    elif resp.search_params['language'] != 'all':
+        search_lang = [lc[3]
                        for lc in language_codes
-                       if lc[0][:2] == resp.search_params['language'].split('_')[0]][0]
+                       if lc[0].split('-')[0] == resp.search_params['language'].split('-')[0]]
+        search_lang = search_lang[0].split(' (')[0]
 
     # parse results
     for result in dom.xpath(results_xpath):
@@ -59,7 +64,7 @@ def response(resp):
         elif search_lang:
             href = href + search_lang + '/'
 
-        title = escape(extract_text(link))
+        title = extract_text(link)
 
         content = extract_text(result.xpath('.//div[contains(@class,"red")]'))
         content = content + " - "
@@ -75,7 +80,7 @@ def response(resp):
         # append result
         results.append({'url': href,
                         'title': title,
-                        'content': escape(content)})
+                        'content': content})
 
     # return results
     return results

searx/engines/swisscows.py

@@ -10,10 +10,10 @@
  @parse       url, title, content
 """
 
-from cgi import escape
 from json import loads
-from urllib import urlencode, unquote
 import re
+from lxml.html import fromstring
+from searx.url_utils import unquote, urlencode
 
 # engine dependent config
 categories = ['general', 'images']
@@ -24,11 +24,13 @@ language_support = True
 base_url = 'https://swisscows.ch/'
 search_string = '?{query}&page={page}'
 
+supported_languages_url = base_url
+
 # regex
-regex_json = re.compile(r'initialData: {"Request":(.|\n)*},\s*environment')
-regex_json_remove_start = re.compile(r'^initialData:\s*')
-regex_json_remove_end = re.compile(r',\s*environment$')
-regex_img_url_remove_start = re.compile(r'^https?://i\.swisscows\.ch/\?link=')
+regex_json = re.compile(b'initialData: {"Request":(.|\n)*},\s*environment')
+regex_json_remove_start = re.compile(b'^initialData:\s*')
+regex_json_remove_end = re.compile(b',\s*environment$')
+regex_img_url_remove_start = re.compile(b'^https?://i\.swisscows\.ch/\?link=')
 
 
 # do search-request
@@ -36,15 +38,16 @@ def request(query, params):
     if params['language'] == 'all':
         ui_language = 'browser'
         region = 'browser'
+    elif params['language'].split('-')[0] == 'no':
+        region = 'nb-NO'
     else:
-        region = params['language'].replace('_', '-')
-        ui_language = params['language'].split('_')[0]
+        region = params['language']
+        ui_language = params['language'].split('-')[0]
 
     search_path = search_string.format(
-        query=urlencode({'query': query,
-                         'uiLanguage': ui_language,
-                         'region': region}),
-        page=params['pageno'])
+        query=urlencode({'query': query, 'uiLanguage': ui_language, 'region': region}),
+        page=params['pageno']
+    )
 
     # image search query is something like 'image?{query}&page={page}'
     if params['category'] == 'images':
@@ -59,14 +62,14 @@ def request(query, params):
 def response(resp):
     results = []
 
-    json_regex = regex_json.search(resp.content)
+    json_regex = regex_json.search(resp.text)
 
     # check if results are returned
     if not json_regex:
         return []
 
-    json_raw = regex_json_remove_end.sub('', regex_json_remove_start.sub('', json_regex.group()))
-    json = loads(json_raw)
+    json_raw = regex_json_remove_end.sub(b'', regex_json_remove_start.sub(b'', json_regex.group()))
+    json = loads(json_raw.decode('utf-8'))
 
     # parse results
     for result in json['Results'].get('items', []):
@@ -74,11 +77,11 @@ def response(resp):
 
         # parse image results
         if result.get('ContentType', '').startswith('image'):
-            img_url = unquote(regex_img_url_remove_start.sub('', result['Url']))
+            img_url = unquote(regex_img_url_remove_start.sub(b'', result['Url'].encode('utf-8')).decode('utf-8'))
 
             # append result
             results.append({'url': result['SourceUrl'],
-                            'title': escape(result['Title']),
+                            'title': result['Title'],
                             'content': '',
                             'img_src': img_url,
                             'template': 'images.html'})
@@ -90,20 +93,34 @@ def response(resp):
 
             # append result
             results.append({'url': result_url,
-                            'title': escape(result_title),
-                            'content': escape(result_content)})
+                            'title': result_title,
+                            'content': result_content})
 
     # parse images
     for result in json.get('Images', []):
         # decode image url
-        img_url = unquote(regex_img_url_remove_start.sub('', result['Url']))
+        img_url = unquote(regex_img_url_remove_start.sub(b'', result['Url'].encode('utf-8')).decode('utf-8'))
 
         # append result
         results.append({'url': result['SourceUrl'],
-                        'title': escape(result['Title']),
+                        'title': result['Title'],
                         'content': '',
                         'img_src': img_url,
                         'template': 'images.html'})
 
     # return results
     return results
+
+
+# get supported languages from their site
+def _fetch_supported_languages(resp):
+    supported_languages = []
+    dom = fromstring(resp.text)
+    options = dom.xpath('//div[@id="regions-popup"]//ul/li/a')
+    for option in options:
+        code = option.xpath('./@data-val')[0]
+        if code.startswith('nb-'):
+            code = code.replace('nb', 'no', 1)
+        supported_languages.append(code)
+
+    return supported_languages

searx/engines/tokyotoshokan.py

@@ -11,12 +11,11 @@
 """
 
 import re
-from cgi import escape
-from urllib import urlencode
 from lxml import html
 from searx.engines.xpath import extract_text
 from datetime import datetime
 from searx.engines.nyaa import int_or_zero, get_filesize_mul
+from searx.url_utils import urlencode
 
 # engine dependent config
 categories = ['files', 'videos', 'music']
@@ -29,8 +28,7 @@ search_url = base_url + 'search.php?{query}'
 
 # do search-request
 def request(query, params):
-    query = urlencode({'page': params['pageno'],
-                       'terms': query})
+    query = urlencode({'page': params['pageno'], 'terms': query})
     params['url'] = search_url.format(query=query)
     return params
 
@@ -51,7 +49,7 @@ def response(resp):
     size_re = re.compile(r'Size:\s*([\d.]+)(TB|GB|MB|B)', re.IGNORECASE)
 
     # processing the results, two rows at a time
-    for i in xrange(0, len(rows), 2):
+    for i in range(0, len(rows), 2):
         # parse the first row
         name_row = rows[i]
 
@@ -80,14 +78,14 @@ def response(resp):
                     groups = size_re.match(item).groups()
                     multiplier = get_filesize_mul(groups[1])
                     params['filesize'] = int(multiplier * float(groups[0]))
-                except Exception as e:
+                except:
                     pass
             elif item.startswith('Date:'):
                 try:
                     # Date: 2016-02-21 21:44 UTC
                     date = datetime.strptime(item, 'Date: %Y-%m-%d %H:%M UTC')
                     params['publishedDate'] = date
-                except Exception as e:
+                except:
                     pass
             elif item.startswith('Comment:'):
                 params['content'] = item

searx/engines/torrentz.py

@@ -12,12 +12,11 @@
 """
 
 import re
-from cgi import escape
-from urllib import urlencode
 from lxml import html
-from searx.engines.xpath import extract_text
 from datetime import datetime
 from searx.engines.nyaa import int_or_zero, get_filesize_mul
+from searx.engines.xpath import extract_text
+from searx.url_utils import urlencode
 
 # engine dependent config
 categories = ['files', 'videos', 'music']
@@ -71,7 +70,7 @@ def response(resp):
             size_str = result.xpath('./dd/span[@class="s"]/text()')[0]
             size, suffix = size_str.split()
             params['filesize'] = int(size) * get_filesize_mul(suffix)
-        except Exception as e:
+        except:
             pass
 
         # does our link contain a valid SHA1 sum?
@@ -85,7 +84,7 @@ def response(resp):
             # Fri, 25 Mar 2016 16:29:01
             date = datetime.strptime(date_str, '%a, %d %b %Y %H:%M:%S')
             params['publishedDate'] = date
-        except Exception as e:
+        except:
             pass
 
         results.append(params)

searx/engines/translated.py

@@ -9,9 +9,12 @@
  @parse       url, title, content
 """
 import re
-from cgi import escape
+from sys import version_info
 from searx.utils import is_valid_lang
 
+if version_info[0] == 3:
+    unicode = str
+
 categories = ['general']
 url = u'http://api.mymemory.translated.net/get?q={query}&langpair={from_lang}|{to_lang}{key}'
 web_url = u'http://mymemory.translated.net/en/{from_lang}/{to_lang}/{query}'
@@ -52,14 +55,14 @@ def request(query, params):
 def response(resp):
     results = []
     results.append({
-        'url': escape(web_url.format(
+        'url': web_url.format(
             from_lang=resp.search_params['from_lang'][2],
             to_lang=resp.search_params['to_lang'][2],
-            query=resp.search_params['query'])),
-        'title': escape('[{0}-{1}] {2}'.format(
+            query=resp.search_params['query']),
+        'title': '[{0}-{1}] {2}'.format(
             resp.search_params['from_lang'][1],
             resp.search_params['to_lang'][1],
-            resp.search_params['query'])),
-        'content': escape(resp.json()['responseData']['translatedText'])
+            resp.search_params['query']),
+        'content': resp.json()['responseData']['translatedText']
     })
     return results

searx/engines/twitter.py

@@ -12,11 +12,10 @@
  @todo        publishedDate
 """
 
-from urlparse import urljoin
-from urllib import urlencode
 from lxml import html
 from datetime import datetime
 from searx.engines.xpath import extract_text
+from searx.url_utils import urlencode, urljoin
 
 # engine dependent config
 categories = ['social media']
@@ -28,6 +27,7 @@ search_url = base_url + 'search?'
 
 # specific xpath variables
 results_xpath = '//li[@data-item-type="tweet"]'
+avatar_xpath = './/img[contains(@class, "avatar")]/@src'
 link_xpath = './/small[@class="time"]//a'
 title_xpath = './/span[contains(@class, "username")]'
 content_xpath = './/p[contains(@class, "tweet-text")]'
@@ -40,7 +40,7 @@ def request(query, params):
 
     # set language if specified
     if params['language'] != 'all':
-        params['cookies']['lang'] = params['language'].split('_')[0]
+        params['cookies']['lang'] = params['language'].split('-')[0]
     else:
         params['cookies']['lang'] = 'en'
 
@@ -58,6 +58,8 @@ def response(resp):
         try:
             link = tweet.xpath(link_xpath)[0]
             content = extract_text(tweet.xpath(content_xpath)[0])
+            img_src = tweet.xpath(avatar_xpath)[0]
+            img_src = img_src.replace('_bigger', '_normal')
         except Exception:
             continue
 
@@ -72,12 +74,14 @@ def response(resp):
             results.append({'url': url,
                             'title': title,
                             'content': content,
+                            'img_src': img_src,
                             'publishedDate': publishedDate})
         else:
             # append result
             results.append({'url': url,
                             'title': title,
-                            'content': content})
+                            'content': content,
+                            'img_src': img_src})
 
     # return results
     return results

searx/engines/vimeo.py

@@ -12,28 +12,19 @@
 # @todo        rewrite to api
 # @todo        set content-parameter with correct data
 
-from urllib import urlencode
-from lxml import html
-from HTMLParser import HTMLParser
-from searx.engines.xpath import extract_text
+from json import loads
 from dateutil import parser
+from searx.url_utils import urlencode
 
 # engine dependent config
 categories = ['videos']
 paging = True
 
 # search-url
-base_url = 'https://vimeo.com'
+base_url = 'https://vimeo.com/'
 search_url = base_url + '/search/page:{pageno}?{query}'
 
-# specific xpath variables
-results_xpath = '//div[contains(@class,"results_grid")]/ul/li'
-url_xpath = './/a/@href'
-title_xpath = './/span[@class="title"]'
-thumbnail_xpath = './/img[@class="js-clip_thumbnail_image"]/@src'
-publishedDate_xpath = './/time/attribute::datetime'
-
-embedded_url = '<iframe data-src="//player.vimeo.com/video{videoid}" ' +\
+embedded_url = '<iframe data-src="//player.vimeo.com/video/{videoid}" ' +\
     'width="540" height="304" frameborder="0" ' +\
     'webkitallowfullscreen mozallowfullscreen allowfullscreen></iframe>'
 
@@ -49,17 +40,18 @@ def request(query, params):
 # get response from search-request
 def response(resp):
     results = []
-
-    dom = html.fromstring(resp.text)
-    p = HTMLParser()
+    data_start_pos = resp.text.find('{"filtered"')
+    data_end_pos = resp.text.find(';\n', data_start_pos + 1)
+    data = loads(resp.text[data_start_pos:data_end_pos])
 
     # parse results
-    for result in dom.xpath(results_xpath):
-        videoid = result.xpath(url_xpath)[0]
+    for result in data['filtered']['data']:
+        result = result[result['type']]
+        videoid = result['uri'].split('/')[-1]
         url = base_url + videoid
-        title = p.unescape(extract_text(result.xpath(title_xpath)))
-        thumbnail = extract_text(result.xpath(thumbnail_xpath)[0])
-        publishedDate = parser.parse(extract_text(result.xpath(publishedDate_xpath)[0]))
+        title = result['name']
+        thumbnail = result['pictures']['sizes'][-1]['link']
+        publishedDate = parser.parse(result['created_time'])
         embedded = embedded_url.format(videoid=videoid)
 
         # append result

searx/engines/wikidata.py

@@ -14,10 +14,11 @@
 from searx import logger
 from searx.poolrequests import get
 from searx.engines.xpath import extract_text
+from searx.engines.wikipedia import _fetch_supported_languages, supported_languages_url
+from searx.url_utils import urlencode
 
 from json import loads
 from lxml.html import fromstring
-from urllib import urlencode
 
 logger = logger.getChild('wikidata')
 result_count = 1
@@ -55,31 +56,29 @@ calendar_name_xpath = './/sup[contains(@class,"wb-calendar-name")]'
 
 
 def request(query, params):
-    language = params['language'].split('_')[0]
+    language = params['language'].split('-')[0]
     if language == 'all':
         language = 'en'
 
     params['url'] = url_search.format(
-        query=urlencode({'label': query,
-                        'language': language}))
+        query=urlencode({'label': query, 'language': language}))
     return params
 
 
 def response(resp):
     results = []
-    html = fromstring(resp.content)
+    html = fromstring(resp.text)
     wikidata_ids = html.xpath(wikidata_ids_xpath)
 
-    language = resp.search_params['language'].split('_')[0]
+    language = resp.search_params['language'].split('-')[0]
     if language == 'all':
         language = 'en'
 
     # TODO: make requests asynchronous to avoid timeout when result_count > 1
     for wikidata_id in wikidata_ids[:result_count]:
-        url = url_detail.format(query=urlencode({'page': wikidata_id,
-                                                'uselang': language}))
+        url = url_detail.format(query=urlencode({'page': wikidata_id, 'uselang': language}))
         htmlresponse = get(url)
-        jsonresponse = loads(htmlresponse.content)
+        jsonresponse = loads(htmlresponse.text)
         results += getDetail(jsonresponse, wikidata_id, language, resp.search_params['language'])
 
     return results

searx/engines/wikipedia.py

@@ -11,11 +11,12 @@
 """
 
 from json import loads
-from urllib import urlencode, quote
+from lxml.html import fromstring
+from searx.url_utils import quote, urlencode
 
 # search-url
-base_url = 'https://{language}.wikipedia.org/'
-search_postfix = 'w/api.php?'\
+base_url = u'https://{language}.wikipedia.org/'
+search_url = base_url + u'w/api.php?'\
     'action=query'\
     '&format=json'\
     '&{query}'\
@@ -24,25 +25,27 @@ search_postfix = 'w/api.php?'\
     '&explaintext'\
     '&pithumbsize=300'\
     '&redirects'
+supported_languages_url = 'https://meta.wikimedia.org/wiki/List_of_Wikipedias'
 
 
 # set language in base_url
 def url_lang(lang):
-    if lang == 'all':
+    lang = lang.split('-')[0]
+    if lang == 'all' or lang not in supported_languages:
         language = 'en'
     else:
-        language = lang.split('_')[0]
+        language = lang
 
-    return base_url.format(language=language)
+    return language
 
 
 # do search-request
 def request(query, params):
     if query.islower():
-        query += '|' + query.title()
+        query = u'{0}|{1}'.format(query.decode('utf-8'), query.decode('utf-8').title()).encode('utf-8')
 
-    params['url'] = url_lang(params['language']) \
-        + search_postfix.format(query=urlencode({'titles': query}))
+    params['url'] = search_url.format(query=urlencode({'titles': query}),
+                                      language=url_lang(params['language']))
 
     return params
 
@@ -74,7 +77,7 @@ def extract_first_paragraph(content, title, image):
 def response(resp):
     results = []
 
-    search_result = loads(resp.content)
+    search_result = loads(resp.text)
 
     # wikipedia article's unique id
     # first valid id is assumed to be the requested article
@@ -95,11 +98,9 @@ def response(resp):
     extract = page.get('extract')
 
     summary = extract_first_paragraph(extract, title, image)
-    if not summary:
-        return []
 
     # link to wikipedia article
-    wikipedia_link = url_lang(resp.search_params['language']) \
+    wikipedia_link = base_url.format(language=url_lang(resp.search_params['language'])) \
         + 'wiki/' + quote(title.replace(' ', '_').encode('utf8'))
 
     results.append({'url': wikipedia_link, 'title': title})
@@ -111,3 +112,24 @@ def response(resp):
                     'urls': [{'title': 'Wikipedia', 'url': wikipedia_link}]})
 
     return results
+
+
+# get supported languages from their site
+def _fetch_supported_languages(resp):
+    supported_languages = {}
+    dom = fromstring(resp.text)
+    tables = dom.xpath('//table[contains(@class,"sortable")]')
+    for table in tables:
+        # exclude header row
+        trs = table.xpath('.//tr')[1:]
+        for tr in trs:
+            td = tr.xpath('./td')
+            code = td[3].xpath('./a')[0].text
+            name = td[2].xpath('./a')[0].text
+            english_name = td[1].xpath('./a')[0].text
+            articles = int(td[4].xpath('./a/b')[0].text.replace(',', ''))
+            # exclude languages with too few articles
+            if articles >= 100:
+                supported_languages[code] = {"name": name, "english_name": english_name, "articles": articles}
+
+    return supported_languages

searx/engines/wolframalpha_api.py

@@ -8,8 +8,8 @@
 # @stable      yes
 # @parse       url, infobox
 
-from urllib import urlencode
 from lxml import etree
+from searx.url_utils import urlencode
 
 # search-url
 search_url = 'https://api.wolframalpha.com/v2/query?appid={api_key}&{query}'
@@ -18,10 +18,10 @@ api_key = ''  # defined in settings.yml
 
 # xpath variables
 failure_xpath = '/queryresult[attribute::success="false"]'
-answer_xpath = '//pod[attribute::primary="true"]/subpod/plaintext'
 input_xpath = '//pod[starts-with(attribute::id, "Input")]/subpod/plaintext'
 pods_xpath = '//pod'
 subpods_xpath = './subpod'
+pod_primary_xpath = './@primary'
 pod_id_xpath = './@id'
 pod_title_xpath = './@title'
 plaintext_xpath = './plaintext'
@@ -37,8 +37,7 @@ image_pods = {'VisualRepresentation',
 
 # do search-request
 def request(query, params):
-    params['url'] = search_url.format(query=urlencode({'input': query}),
-                                      api_key=api_key)
+    params['url'] = search_url.format(query=urlencode({'input': query}), api_key=api_key)
     params['headers']['Referer'] = site_url.format(query=urlencode({'i': query}))
 
     return params
@@ -56,7 +55,7 @@ def replace_pua_chars(text):
                  u'\uf74e': 'i',        # imaginary number
                  u'\uf7d9': '='}        # equals sign
 
-    for k, v in pua_chars.iteritems():
+    for k, v in pua_chars.items():
         text = text.replace(k, v)
 
     return text
@@ -66,7 +65,7 @@ def replace_pua_chars(text):
 def response(resp):
     results = []
 
-    search_results = etree.XML(resp.content)
+    search_results = etree.XML(resp.text)
 
     # return empty array if there are no results
     if search_results.xpath(failure_xpath):
@@ -75,13 +74,15 @@ def response(resp):
     try:
         infobox_title = search_results.xpath(input_xpath)[0].text
     except:
-        infobox_title = None
+        infobox_title = ""
 
     pods = search_results.xpath(pods_xpath)
     result_chunks = []
+    result_content = ""
     for pod in pods:
         pod_id = pod.xpath(pod_id_xpath)[0]
         pod_title = pod.xpath(pod_title_xpath)[0]
+        pod_is_result = pod.xpath(pod_primary_xpath)
 
         subpods = pod.xpath(subpods_xpath)
         if not subpods:
@@ -94,6 +95,10 @@ def response(resp):
 
             if content and pod_id not in image_pods:
 
+                if pod_is_result or not result_content:
+                    if pod_id != "Input":
+                        result_content = "%s: %s" % (pod_title, content)
+
                 # if no input pod was found, title is first plaintext pod
                 if not infobox_title:
                     infobox_title = content
@@ -109,14 +114,16 @@ def response(resp):
     if not result_chunks:
         return []
 
+    title = "Wolfram|Alpha (%s)" % infobox_title
+
     # append infobox
     results.append({'infobox': infobox_title,
                     'attributes': result_chunks,
-                    'urls': [{'title': 'Wolfram|Alpha', 'url': resp.request.headers['Referer'].decode('utf8')}]})
+                    'urls': [{'title': 'Wolfram|Alpha', 'url': resp.request.headers['Referer']}]})
 
     # append link to site
-    results.append({'url': resp.request.headers['Referer'].decode('utf8'),
-                    'title': 'Wolfram|Alpha',
-                    'content': infobox_title})
+    results.append({'url': resp.request.headers['Referer'],
+                    'title': title,
+                    'content': result_content})
 
     return results

searx/engines/wolframalpha_noapi.py

@@ -10,9 +10,9 @@
 
 from json import loads
 from time import time
-from urllib import urlencode
 
 from searx.poolrequests import get as http_get
+from searx.url_utils import urlencode
 
 # search-url
 url = 'https://www.wolframalpha.com/'
@@ -34,7 +34,7 @@ search_url = url + 'input/json.jsp'\
 referer_url = url + 'input/?{query}'
 
 token = {'value': '',
-         'last_updated': 0}
+         'last_updated': None}
 
 # pods to display as image in infobox
 # this pods do return a plaintext, but they look better and are more useful as images
@@ -61,7 +61,7 @@ obtain_token()
 # do search-request
 def request(query, params):
     # obtain token if last update was more than an hour
-    if time() - token['last_updated'] > 3600:
+    if time() - (token['last_updated'] or 0) > 3600:
         obtain_token()
     params['url'] = search_url.format(query=urlencode({'input': query}), token=token['value'])
     params['headers']['Referer'] = referer_url.format(query=urlencode({'i': query}))
@@ -80,10 +80,12 @@ def response(resp):
 
     # TODO handle resp_json['queryresult']['assumptions']
     result_chunks = []
-    infobox_title = None
+    infobox_title = ""
+    result_content = ""
     for pod in resp_json['queryresult']['pods']:
         pod_id = pod.get('id', '')
         pod_title = pod.get('title', '')
+        pod_is_result = pod.get('primary', None)
 
         if 'subpods' not in pod:
             continue
@@ -97,6 +99,10 @@ def response(resp):
                 if subpod['plaintext'] != '(requires interactivity)':
                     result_chunks.append({'label': pod_title, 'value': subpod['plaintext']})
 
+                if pod_is_result or not result_content:
+                    if pod_id != "Input":
+                        result_content = pod_title + ': ' + subpod['plaintext']
+
             elif 'img' in subpod:
                 result_chunks.append({'label': pod_title, 'image': subpod['img']})
 
@@ -105,10 +111,10 @@ def response(resp):
 
     results.append({'infobox': infobox_title,
                     'attributes': result_chunks,
-                    'urls': [{'title': 'Wolfram|Alpha', 'url': resp.request.headers['Referer'].decode('utf8')}]})
+                    'urls': [{'title': 'Wolfram|Alpha', 'url': resp.request.headers['Referer']}]})
 
-    results.append({'url': resp.request.headers['Referer'].decode('utf8'),
-                    'title': 'Wolfram|Alpha',
-                    'content': infobox_title})
+    results.append({'url': resp.request.headers['Referer'],
+                    'title': 'Wolfram|Alpha (' + infobox_title + ')',
+                    'content': result_content})
 
     return results

searx/engines/www1x.py

@@ -10,11 +10,9 @@
  @parse       url, title, thumbnail, img_src, content
 """
 
-from urllib import urlencode
-from urlparse import urljoin
 from lxml import html
-import string
 import re
+from searx.url_utils import urlencode, urljoin
 
 # engine dependent config
 categories = ['images']
@@ -55,7 +53,7 @@ def response(resp):
         cur_element += result_part
 
         # fix xml-error
-        cur_element = string.replace(cur_element, '"></a>', '"/></a>')
+        cur_element = cur_element.replace('"></a>', '"/></a>')
 
         dom = html.fromstring(cur_element)
         link = dom.xpath('//a')[0]

searx/engines/www500px.py

@@ -12,12 +12,8 @@
  @todo        rewrite to api
 """
 
-
-from urllib import urlencode
-from urlparse import urljoin
-from lxml import html
-import re
-from searx.engines.xpath import extract_text
+from json import loads
+from searx.url_utils import urlencode, urljoin
 
 # engine dependent config
 categories = ['images']
@@ -25,13 +21,27 @@ paging = True
 
 # search-url
 base_url = 'https://500px.com'
-search_url = base_url + '/search?search?page={pageno}&type=photos&{query}'
+search_url = 'https://api.500px.com/v1/photos/search?type=photos'\
+    '&{query}'\
+    '&image_size%5B%5D=4'\
+    '&image_size%5B%5D=20'\
+    '&image_size%5B%5D=21'\
+    '&image_size%5B%5D=1080'\
+    '&image_size%5B%5D=1600'\
+    '&image_size%5B%5D=2048'\
+    '&include_states=true'\
+    '&formats=jpeg%2Clytro'\
+    '&include_tags=true'\
+    '&exclude_nude=true'\
+    '&page={pageno}'\
+    '&rpp=50'\
+    '&sdk_key=b68e60cff4c929bedea36ca978830c5caca790c3'
 
 
 # do search-request
 def request(query, params):
     params['url'] = search_url.format(pageno=params['pageno'],
-                                      query=urlencode({'q': query}))
+                                      query=urlencode({'term': query}))
 
     return params
 
@@ -40,19 +50,16 @@ def request(query, params):
 def response(resp):
     results = []
 
-    dom = html.fromstring(resp.text)
-    regex = re.compile(r'3\.jpg.*$')
+    response_json = loads(resp.text)
 
     # parse results
-    for result in dom.xpath('//div[@class="photo"]'):
-        link = result.xpath('.//a')[0]
-        url = urljoin(base_url, link.attrib.get('href'))
-        title = extract_text(result.xpath('.//div[@class="title"]'))
-        thumbnail_src = link.xpath('.//img')[0].attrib.get('src')
-        # To have a bigger thumbnail, uncomment the next line
-        # thumbnail_src = regex.sub('4.jpg', thumbnail_src)
-        content = extract_text(result.xpath('.//div[@class="info"]'))
-        img_src = regex.sub('2048.jpg', thumbnail_src)
+    for result in response_json['photos']:
+        url = urljoin(base_url, result['url'])
+        title = result['name']
+        # last index is the biggest resolution
+        img_src = result['image_url'][-1]
+        thumbnail_src = result['image_url'][0]
+        content = result['description'] or ''
 
         # append result
         results.append({'url': url,

searx/engines/xpath.py

@@ -1,13 +1,13 @@
 from lxml import html
-from urllib import urlencode, unquote
-from urlparse import urlparse, urljoin
 from lxml.etree import _ElementStringResult, _ElementUnicodeResult
 from searx.utils import html_to_text
+from searx.url_utils import unquote, urlencode, urljoin, urlparse
 
 search_url = None
 url_xpath = None
 content_xpath = None
 title_xpath = None
+paging = False
 suggestion_xpath = ''
 results_xpath = ''
 
@@ -31,8 +31,6 @@ if xpath_results is a string element, then it's already done
 def extract_text(xpath_results):
     if type(xpath_results) == list:
         # it's list of result : concat everything using recursive call
-        if not xpath_results:
-            raise Exception('Empty url resultset')
         result = ''
         for e in xpath_results:
             result = result + extract_text(e)
@@ -42,16 +40,20 @@ def extract_text(xpath_results):
         return ''.join(xpath_results)
     else:
         # it's a element
-        return html_to_text(xpath_results.text_content()).strip()
+        text = html.tostring(xpath_results, encoding='unicode', method='text', with_tail=False)
+        text = text.strip().replace('\n', ' ')
+        return ' '.join(text.split())
 
 
 def extract_url(xpath_results, search_url):
+    if xpath_results == []:
+        raise Exception('Empty url resultset')
     url = extract_text(xpath_results)
 
     if url.startswith('//'):
         # add http or https to this kind of url //example.com/
         parsed_search_url = urlparse(search_url)
-        url = parsed_search_url.scheme + url
+        url = u'{0}:{1}'.format(parsed_search_url.scheme, url)
     elif url.startswith('/'):
         # fix relative url to the search engine
         url = urljoin(search_url, url)
@@ -101,8 +103,8 @@ def response(resp):
     if results_xpath:
         for result in dom.xpath(results_xpath):
             url = extract_url(result.xpath(url_xpath), search_url)
-            title = extract_text(result.xpath(title_xpath)[0])
-            content = extract_text(result.xpath(content_xpath)[0])
+            title = extract_text(result.xpath(title_xpath))
+            content = extract_text(result.xpath(content_xpath))
             results.append({'url': url, 'title': title, 'content': content})
     else:
         for url, title, content in zip(

searx/engines/yacy.py

@@ -13,8 +13,10 @@
 # @todo        parse video, audio and file results
 
 from json import loads
-from urllib import urlencode
 from dateutil import parser
+from searx.url_utils import urlencode
+
+from searx.utils import html_to_text
 
 # engine dependent config
 categories = ['general', 'images']  # TODO , 'music', 'videos', 'files'
@@ -51,7 +53,7 @@ def request(query, params):
 
     # add language tag if specified
     if params['language'] != 'all':
-        params['url'] += '&lr=lang_' + params['language'].split('_')[0]
+        params['url'] += '&lr=lang_' + params['language'].split('-')[0]
 
     return params
 
@@ -88,7 +90,7 @@ def response(resp):
             # append result
             results.append({'url': result['link'],
                             'title': result['title'],
-                            'content': result['description'],
+                            'content': html_to_text(result['description']),
                             'publishedDate': publishedDate})
 
         # TODO parse video, audio and file results

searx/engines/yahoo.py

@@ -11,10 +11,9 @@
  @parse       url, title, content, suggestion
 """
 
-from urllib import urlencode
-from urlparse import unquote
 from lxml import html
 from searx.engines.xpath import extract_text, extract_url
+from searx.url_utils import unquote, urlencode
 
 # engine dependent config
 categories = ['general']
@@ -27,6 +26,8 @@ base_url = 'https://search.yahoo.com/'
 search_url = 'search?{query}&b={offset}&fl=1&vl=lang_{lang}'
 search_url_with_time = 'search?{query}&b={offset}&fl=1&vl=lang_{lang}&age={age}&btf={btf}&fr2=time'
 
+supported_languages_url = 'https://search.yahoo.com/web/advanced'
+
 # specific xpath variables
 results_xpath = "//div[contains(concat(' ', normalize-space(@class), ' '), ' Sr ')]"
 url_xpath = './/h3/a/@href'
@@ -72,11 +73,20 @@ def _get_url(query, offset, language, time_range):
 def _get_language(params):
     if params['language'] == 'all':
         return 'en'
-    return params['language'].split('_')[0]
+    elif params['language'][:2] == 'zh':
+        if params['language'] == 'zh' or params['language'] == 'zh-CH':
+            return 'szh'
+        else:
+            return 'tzh'
+    else:
+        return params['language'].split('-')[0]
 
 
 # do search-request
 def request(query, params):
+    if params['time_range'] and params['time_range'] not in time_range_dict:
+        return params
+
     offset = (params['pageno'] - 1) * 10 + 1
     language = _get_language(params)
 
@@ -129,3 +139,15 @@ def response(resp):
 
     # return results
     return results
+
+
+# get supported languages from their site
+def _fetch_supported_languages(resp):
+    supported_languages = []
+    dom = html.fromstring(resp.text)
+    options = dom.xpath('//div[@id="yschlang"]/span/label/input')
+    for option in options:
+        code = option.xpath('./@value')[0][5:].replace('_', '-')
+        supported_languages.append(code)
+
+    return supported_languages

searx/engines/yahoo_news.py

@@ -9,13 +9,13 @@
 # @stable      no (HTML can change)
 # @parse       url, title, content, publishedDate
 
-from urllib import urlencode
+import re
+from datetime import datetime, timedelta
 from lxml import html
 from searx.engines.xpath import extract_text, extract_url
-from searx.engines.yahoo import parse_url
-from datetime import datetime, timedelta
-import re
+from searx.engines.yahoo import parse_url, _fetch_supported_languages, supported_languages_url
 from dateutil import parser
+from searx.url_utils import urlencode
 
 # engine dependent config
 categories = ['news']
@@ -80,16 +80,19 @@ def response(resp):
 
         # still useful ?
         if re.match("^[0-9]+ minute(s|) ago$", publishedDate):
-            publishedDate = datetime.now() - timedelta(minutes=int(re.match(r'\d+', publishedDate).group()))  # noqa
+            publishedDate = datetime.now() - timedelta(minutes=int(re.match(r'\d+', publishedDate).group()))
+        elif re.match("^[0-9]+ days? ago$", publishedDate):
+            publishedDate = datetime.now() - timedelta(days=int(re.match(r'\d+', publishedDate).group()))
+        elif re.match("^[0-9]+ hour(s|), [0-9]+ minute(s|) ago$", publishedDate):
+            timeNumbers = re.findall(r'\d+', publishedDate)
+            publishedDate = datetime.now()\
+                - timedelta(hours=int(timeNumbers[0]))\
+                - timedelta(minutes=int(timeNumbers[1]))
         else:
-            if re.match("^[0-9]+ hour(s|), [0-9]+ minute(s|) ago$",
-                        publishedDate):
-                timeNumbers = re.findall(r'\d+', publishedDate)
-                publishedDate = datetime.now()\
-                    - timedelta(hours=int(timeNumbers[0]))\
-                    - timedelta(minutes=int(timeNumbers[1]))
-            else:
+            try:
                 publishedDate = parser.parse(publishedDate)
+            except:
+                publishedDate = datetime.now()
 
         if publishedDate.year == 1900:
             publishedDate = publishedDate.replace(year=datetime.now().year)

searx/engines/yandex.py

@@ -9,10 +9,9 @@
  @parse       url, title, content
 """
 
-from cgi import escape
-from urllib import urlencode
 from lxml import html
-from searx.search import logger
+from searx import logger
+from searx.url_utils import urlencode
 
 logger = logger.getChild('yandex engine')
 
@@ -23,21 +22,23 @@ language_support = True  # TODO
 
 default_tld = 'com'
 language_map = {'ru': 'ru',
-                'ua': 'uk',
+                'ua': 'ua',
+                'be': 'by',
+                'kk': 'kz',
                 'tr': 'com.tr'}
 
 # search-url
 base_url = 'https://yandex.{tld}/'
 search_url = 'search/?{query}&p={page}'
 
-results_xpath = '//div[@class="serp-item serp-item_plain_yes clearfix i-bem"]'
+results_xpath = '//li[@class="serp-item"]'
 url_xpath = './/h2/a/@href'
 title_xpath = './/h2/a//text()'
-content_xpath = './/div[@class="serp-item__text"]//text()'
+content_xpath = './/div[@class="text-container typo typo_text_m typo_line_m organic__text"]//text()'
 
 
 def request(query, params):
-    lang = params['language'].split('_')[0]
+    lang = params['language'].split('-')[0]
     host = base_url.format(tld=language_map.get(lang) or default_tld)
     params['url'] = host + search_url.format(page=params['pageno'] - 1,
                                              query=urlencode({'text': query}))
@@ -52,8 +53,8 @@ def response(resp):
     for result in dom.xpath(results_xpath):
         try:
             res = {'url': result.xpath(url_xpath)[0],
-                   'title': escape(''.join(result.xpath(title_xpath))),
-                   'content': escape(''.join(result.xpath(content_xpath)))}
+                   'title': ''.join(result.xpath(title_xpath)),
+                   'content': ''.join(result.xpath(content_xpath))}
         except:
             logger.exception('yandex parse crash')
             continue

searx/engines/youtube_api.py

@@ -9,8 +9,8 @@
 # @parse       url, title, content, publishedDate, thumbnail, embedded
 
 from json import loads
-from urllib import urlencode
 from dateutil import parser
+from searx.url_utils import urlencode
 
 # engine dependent config
 categories = ['videos', 'music']
@@ -36,7 +36,7 @@ def request(query, params):
 
     # add language tag if specified
     if params['language'] != 'all':
-        params['url'] += '&relevanceLanguage=' + params['language'].split('_')[0]
+        params['url'] += '&relevanceLanguage=' + params['language'].split('-')[0]
 
     return params
 

searx/engines/youtube_noapi.py

@@ -8,19 +8,25 @@
 # @stable      no
 # @parse       url, title, content, publishedDate, thumbnail, embedded
 
-from urllib import quote_plus
 from lxml import html
 from searx.engines.xpath import extract_text
 from searx.utils import list_get
+from searx.url_utils import quote_plus
 
 # engine dependent config
 categories = ['videos', 'music']
 paging = True
 language_support = False
+time_range_support = True
 
 # search-url
 base_url = 'https://www.youtube.com/results'
 search_url = base_url + '?search_query={query}&page={page}'
+time_range_url = '&sp=EgII{time_range}%253D%253D'
+time_range_dict = {'day': 'Ag',
+                   'week': 'Aw',
+                   'month': 'BA',
+                   'year': 'BQ'}
 
 embedded_url = '<iframe width="540" height="304" ' +\
     'data-src="//www.youtube-nocookie.com/embed/{videoid}" ' +\
@@ -47,6 +53,8 @@ def extract_text_from_dom(result, xpath):
 def request(query, params):
     params['url'] = search_url.format(query=quote_plus(query),
                                       page=params['pageno'])
+    if params['time_range'] in time_range_dict:
+        params['url'] += time_range_url.format(time_range=time_range_dict[params['time_range']])
 
     return params
 

searx/exceptions.py

@@ -0,0 +1,32 @@
+'''
+searx is free software: you can redistribute it and/or modify
+it under the terms of the GNU Affero General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+searx is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU Affero General Public License for more details.
+
+You should have received a copy of the GNU Affero General Public License
+along with searx. If not, see < http://www.gnu.org/licenses/ >.
+
+(C) 2017- by Alexandre Flament, <alex@al-f.net>
+'''
+
+
+class SearxException(Exception):
+    pass
+
+
+class SearxParameterException(SearxException):
+
+    def __init__(self, name, value):
+        if value == '' or value is None:
+            message = 'Empty ' + name + ' parameter'
+        else:
+            message = 'Invalid value "' + value + '" for parameter ' + name
+        super(SearxParameterException, self).__init__(message)
+        self.parameter_name = name
+        self.parameter_value = value

searx/languages.py

@@ -1,78 +1,77 @@
-'''
-searx is free software: you can redistribute it and/or modify
-it under the terms of the GNU Affero General Public License as published by
-the Free Software Foundation, either version 3 of the License, or
-(at your option) any later version.
-
-searx is distributed in the hope that it will be useful,
-but WITHOUT ANY WARRANTY; without even the implied warranty of
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-GNU Affero General Public License for more details.
-
-You should have received a copy of the GNU Affero General Public License
-along with searx. If not, see < http://www.gnu.org/licenses/ >.
-
-(C) 2013- by Adam Tauber, <asciimoo@gmail.com>
-'''
-
+# -*- coding: utf-8 -*-
 # list of language codes
+# this file is generated automatically by utils/update_search_languages.py
+
 language_codes = (
-    ("ar_XA", "Arabic", "Arabia"),
-    ("bg_BG", "Bulgarian", "Bulgaria"),
-    ("cs_CZ", "Czech", "Czech Republic"),
-    ("da_DK", "Danish", "Denmark"),
-    ("de_AT", "German", "Austria"),
-    ("de_CH", "German", "Switzerland"),
-    ("de_DE", "German", "Germany"),
-    ("el_GR", "Greek", "Greece"),
-    ("en_AU", "English", "Australia"),
-    ("en_CA", "English", "Canada"),
-    ("en_GB", "English", "United Kingdom"),
-    ("en_ID", "English", "Indonesia"),
-    ("en_IE", "English", "Ireland"),
-    ("en_IN", "English", "India"),
-    ("en_MY", "English", "Malaysia"),
-    ("en_NZ", "English", "New Zealand"),
-    ("en_PH", "English", "Philippines"),
-    ("en_SG", "English", "Singapore"),
-    ("en_US", "English", "United States"),
-    ("en_XA", "English", "Arabia"),
-    ("en_ZA", "English", "South Africa"),
-    ("es_AR", "Spanish", "Argentina"),
-    ("es_CL", "Spanish", "Chile"),
-    ("es_ES", "Spanish", "Spain"),
-    ("es_MX", "Spanish", "Mexico"),
-    ("es_US", "Spanish", "United States"),
-    ("es_XL", "Spanish", "Latin America"),
-    ("et_EE", "Estonian", "Estonia"),
-    ("fi_FI", "Finnish", "Finland"),
-    ("fr_BE", "French", "Belgium"),
-    ("fr_CA", "French", "Canada"),
-    ("fr_CH", "French", "Switzerland"),
-    ("fr_FR", "French", "France"),
-    ("he_IL", "Hebrew", "Israel"),
-    ("hr_HR", "Croatian", "Croatia"),
-    ("hu_HU", "Hungarian", "Hungary"),
-    ("it_IT", "Italian", "Italy"),
-    ("ja_JP", "Japanese", "Japan"),
-    ("ko_KR", "Korean", "Korea"),
-    ("lt_LT", "Lithuanian", "Lithuania"),
-    ("lv_LV", "Latvian", "Latvia"),
-    ("nb_NO", "Norwegian", "Norway"),
-    ("nl_BE", "Dutch", "Belgium"),
-    ("nl_NL", "Dutch", "Netherlands"),
-    ("oc_OC", "Occitan", "Occitan"),
-    ("pl_PL", "Polish", "Poland"),
-    ("pt_BR", "Portuguese", "Brazil"),
-    ("pt_PT", "Portuguese", "Portugal"),
-    ("ro_RO", "Romanian", "Romania"),
-    ("ru_RU", "Russian", "Russia"),
-    ("sk_SK", "Slovak", "Slovak Republic"),
-    ("sl_SL", "Slovenian", "Slovenia"),
-    ("sv_SE", "Swedish", "Sweden"),
-    ("th_TH", "Thai", "Thailand"),
-    ("tr_TR", "Turkish", "Turkey"),
-    ("uk_UA", "Ukrainian", "Ukraine"),
-    ("zh_CN", "Chinese", "China"),
-    ("zh_HK", "Chinese", "Hong Kong SAR"),
-    ("zh_TW", "Chinese", "Taiwan"))
+    (u"ar-SA", u"العربية", u"", u"Arabic"),
+    (u"bg-BG", u"Български", u"", u"Bulgarian"),
+    (u"cs-CZ", u"Čeština", u"", u"Czech"),
+    (u"da-DK", u"Dansk", u"", u"Danish"),
+    (u"de", u"Deutsch", u"", u"German"),
+    (u"de-AT", u"Deutsch", u"Österreich", u"German"),
+    (u"de-CH", u"Deutsch", u"Schweiz", u"German"),
+    (u"de-DE", u"Deutsch", u"Deutschland", u"German"),
+    (u"el-GR", u"Ελληνικά", u"", u"Greek"),
+    (u"en", u"English", u"", u"English"),
+    (u"en-AU", u"English", u"Australia", u"English"),
+    (u"en-CA", u"English", u"Canada", u"English"),
+    (u"en-CY", u"English", u"Cyprus", u"English"),
+    (u"en-GB", u"English", u"United Kingdom", u"English"),
+    (u"en-GD", u"English", u"Grenada", u"English"),
+    (u"en-ID", u"English", u"Indonesia", u"English"),
+    (u"en-IE", u"English", u"Ireland", u"English"),
+    (u"en-IN", u"English", u"India", u"English"),
+    (u"en-MY", u"English", u"Malaysia", u"English"),
+    (u"en-NZ", u"English", u"New Zealand", u"English"),
+    (u"en-PH", u"English", u"Philippines", u"English"),
+    (u"en-SG", u"English", u"Singapore", u"English"),
+    (u"en-US", u"English", u"United States", u"English"),
+    (u"en-ZA", u"English", u"South Africa", u"English"),
+    (u"es", u"Español", u"", u"Spanish"),
+    (u"es-AR", u"Español", u"Argentina", u"Spanish"),
+    (u"es-CL", u"Español", u"Chile", u"Spanish"),
+    (u"es-CO", u"Español", u"Colombia", u"Spanish"),
+    (u"es-ES", u"Español", u"España", u"Spanish"),
+    (u"es-MX", u"Español", u"México", u"Spanish"),
+    (u"es-PE", u"Español", u"Perú", u"Spanish"),
+    (u"es-US", u"Español", u"Estados Unidos", u"Spanish"),
+    (u"et-EE", u"Eesti", u"", u"Estonian"),
+    (u"fi-FI", u"Suomi", u"", u"Finnish"),
+    (u"fr", u"Français", u"", u"French"),
+    (u"fr-BE", u"Français", u"Belgique", u"French"),
+    (u"fr-CA", u"Français", u"Canada", u"French"),
+    (u"fr-CH", u"Français", u"Suisse", u"French"),
+    (u"fr-FR", u"Français", u"France", u"French"),
+    (u"he-IL", u"עברית", u"", u"Hebrew"),
+    (u"hr-HR", u"Hrvatski", u"", u"Croatian"),
+    (u"hu-HU", u"Magyar", u"", u"Hungarian"),
+    (u"id-ID", u"Bahasa Indonesia", u"", u"Indonesian"),
+    (u"it", u"Italiano", u"", u"Italian"),
+    (u"it-CH", u"Italiano", u"Svizzera", u"Italian"),
+    (u"it-IT", u"Italiano", u"Italia", u"Italian"),
+    (u"ja-JP", u"日本語", u"", u"Japanese"),
+    (u"ko-KR", u"한국어", u"", u"Korean"),
+    (u"lt-LT", u"Lietuvių", u"", u"Lithuanian"),
+    (u"lv-LV", u"Latviešu", u"", u"Latvian"),
+    (u"ms-MY", u"Bahasa Melayu", u"", u"Malay"),
+    (u"nl", u"Nederlands", u"", u"Dutch"),
+    (u"nl-BE", u"Nederlands", u"België", u"Dutch"),
+    (u"nl-NL", u"Nederlands", u"Nederland", u"Dutch"),
+    (u"no-NO", u"Norsk", u"", u"Norwegian"),
+    (u"pl-PL", u"Polski", u"", u"Polish"),
+    (u"pt", u"Português", u"", u"Portuguese"),
+    (u"pt-BR", u"Português", u"Brasil", u"Portuguese"),
+    (u"pt-PT", u"Português", u"Portugal", u"Portuguese"),
+    (u"ro-RO", u"Română", u"", u"Romanian"),
+    (u"ru-RU", u"Русский", u"", u"Russian"),
+    (u"sk-SK", u"Slovenčina", u"", u"Slovak"),
+    (u"sl", u"Slovenščina", u"", u"Slovenian"),
+    (u"sv-SE", u"Svenska", u"", u"Swedish"),
+    (u"th-TH", u"ไทย", u"", u"Thai"),
+    (u"tr-TR", u"Türkçe", u"", u"Turkish"),
+    (u"vi-VN", u"Tiếng Việt", u"", u"Vietnamese"),
+    (u"zh", u"中文", u"", u"Chinese"),
+    (u"zh-CN", u"中文", u"中国", u"Chinese"),
+    (u"zh-HK", u"中文", u"香港", u"Chinese"),
+    (u"zh-TW", u"中文", u"台湾", u"Chinese")
+)

searx/plugins/__init__.py

@@ -14,9 +14,12 @@ along with searx. If not, see < http://www.gnu.org/licenses/ >.
 
 (C) 2015 by Adam Tauber, <asciimoo@gmail.com>
 '''
-from sys import exit
+from sys import exit, version_info
 from searx import logger
 
+if version_info[0] == 3:
+    unicode = str
+
 logger = logger.getChild('plugins')
 
 from searx.plugins import (doai_rewrite,
@@ -63,9 +66,9 @@ class PluginStore():
             plugin.id = plugin.name.replace(' ', '_')
             self.plugins.append(plugin)
 
-    def call(self, plugin_type, request, *args, **kwargs):
+    def call(self, ordered_plugin_list, plugin_type, request, *args, **kwargs):
         ret = True
-        for plugin in request.user_plugins:
+        for plugin in ordered_plugin_list:
             if hasattr(plugin, plugin_type):
                 ret = getattr(plugin, plugin_type)(request, *args, **kwargs)
                 if not ret:

searx/plugins/doai_rewrite.py

@@ -1,6 +1,6 @@
 from flask_babel import gettext
 import re
-from urlparse import urlparse, parse_qsl
+from searx.url_utils import urlparse, parse_qsl
 
 regex = re.compile(r'10\.\d{4,9}/[^\s]+')
 
@@ -20,12 +20,12 @@ def extract_doi(url):
     return None
 
 
-def on_result(request, ctx):
-    doi = extract_doi(ctx['result']['parsed_url'])
+def on_result(request, search, result):
+    doi = extract_doi(result['parsed_url'])
     if doi and len(doi) < 50:
         for suffix in ('/', '.pdf', '/full', '/meta', '/abstract'):
             if doi.endswith(suffix):
                 doi = doi[:-len(suffix)]
-        ctx['result']['url'] = 'http://doai.io/' + doi
-        ctx['result']['parsed_url'] = urlparse(ctx['result']['url'])
+        result['url'] = 'http://doai.io/' + doi
+        result['parsed_url'] = urlparse(result['url'])
     return True

searx/plugins/https_rewrite.py

@@ -16,14 +16,17 @@ along with searx. If not, see < http://www.gnu.org/licenses/ >.
 '''
 
 import re
-from urlparse import urlparse
+import sys
 from lxml import etree
 from os import listdir, environ
 from os.path import isfile, isdir, join
 from searx.plugins import logger
 from flask_babel import gettext
 from searx import searx_dir
+from searx.url_utils import urlparse
 
+if sys.version_info[0] == 3:
+    unicode = str
 
 name = "HTTPS rewrite"
 description = gettext('Rewrite HTTP links to HTTPS if possible')
@@ -220,8 +223,7 @@ def https_url_rewrite(result):
     return result
 
 
-def on_result(request, ctx):
-    result = ctx['result']
+def on_result(request, search, result):
     if result['parsed_url'].scheme == 'http':
         https_url_rewrite(result)
     return True

searx/plugins/self_info.py

@@ -22,25 +22,25 @@ default_on = True
 
 
 # Self User Agent regex
-p = re.compile('.*user[ -]agent.*', re.IGNORECASE)
+p = re.compile(b'.*user[ -]agent.*', re.IGNORECASE)
 
 
 # attach callback to the post search hook
 #  request: flask request object
 #  ctx: the whole local context of the pre search hook
-def post_search(request, ctx):
-    if ctx['search'].pageno > 1:
+def post_search(request, search):
+    if search.search_query.pageno > 1:
         return True
-    if ctx['search'].query == 'ip':
+    if search.search_query.query == b'ip':
         x_forwarded_for = request.headers.getlist("X-Forwarded-For")
         if x_forwarded_for:
             ip = x_forwarded_for[0]
         else:
             ip = request.remote_addr
-        ctx['search'].result_container.answers.clear()
-        ctx['search'].result_container.answers.add(ip)
-    elif p.match(ctx['search'].query):
+        search.result_container.answers.clear()
+        search.result_container.answers.add(ip)
+    elif p.match(search.search_query.query):
         ua = request.user_agent
-        ctx['search'].result_container.answers.clear()
-        ctx['search'].result_container.answers.add(ua)
+        search.result_container.answers.clear()
+        search.result_container.answers.add(ua)
     return True

searx/plugins/tracker_url_remover.py

@@ -17,7 +17,7 @@ along with searx. If not, see < http://www.gnu.org/licenses/ >.
 
 from flask_babel import gettext
 import re
-from urlparse import urlunparse
+from searx.url_utils import urlunparse
 
 regexes = {re.compile(r'utm_[^&]+&?'),
            re.compile(r'(wkey|wemail)[^&]+&?'),
@@ -28,8 +28,8 @@ description = gettext('Remove trackers arguments from the returned URL')
 default_on = True
 
 
-def on_result(request, ctx):
-    query = ctx['result']['parsed_url'].query
+def on_result(request, search, result):
+    query = result['parsed_url'].query
 
     if query == "":
         return True
@@ -37,8 +37,8 @@ def on_result(request, ctx):
     for reg in regexes:
         query = reg.sub('', query)
 
-    if query != ctx['result']['parsed_url'].query:
-        ctx['result']['parsed_url'] = ctx['result']['parsed_url']._replace(query=query)
-        ctx['result']['url'] = urlunparse(ctx['result']['parsed_url'])
+    if query != result['parsed_url'].query:
+        result['parsed_url'] = result['parsed_url']._replace(query=query)
+        result['url'] = urlunparse(result['parsed_url'])
 
     return True

searx/preferences.py

@@ -23,7 +23,7 @@ class Setting(object):
     def __init__(self, default_value, **kwargs):
         super(Setting, self).__init__()
         self.value = default_value
-        for key, value in kwargs.iteritems():
+        for key, value in kwargs.items():
             setattr(self, key, value)
 
         self._post_init()
@@ -38,7 +38,7 @@ class Setting(object):
         return self.value
 
     def save(self, name, resp):
-        resp.set_cookie(name, bytes(self.value), max_age=COOKIE_MAX_AGE)
+        resp.set_cookie(name, self.value, max_age=COOKIE_MAX_AGE)
 
 
 class StringSetting(Setting):
@@ -49,28 +49,32 @@ class StringSetting(Setting):
 class EnumStringSetting(Setting):
     """Setting of a value which can only come from the given choices"""
 
+    def _validate_selection(self, selection):
+        if selection not in self.choices:
+            raise ValidationException('Invalid value: "{0}"'.format(selection))
+
     def _post_init(self):
         if not hasattr(self, 'choices'):
             raise MissingArgumentException('Missing argument: choices')
-
-        if self.value != '' and self.value not in self.choices:
-            raise ValidationException('Invalid default value: {0}'.format(self.value))
+        self._validate_selection(self.value)
 
     def parse(self, data):
-        if data not in self.choices and data != self.value:
-            raise ValidationException('Invalid choice: {0}'.format(data))
+        self._validate_selection(data)
         self.value = data
 
 
 class MultipleChoiceSetting(EnumStringSetting):
     """Setting of values which can only come from the given choices"""
 
+    def _validate_selections(self, selections):
+        for item in selections:
+            if item not in self.choices:
+                raise ValidationException('Invalid value: "{0}"'.format(selections))
+
     def _post_init(self):
         if not hasattr(self, 'choices'):
             raise MissingArgumentException('Missing argument: choices')
-        for item in self.value:
-            if item not in self.choices:
-                raise ValidationException('Invalid default value: {0}'.format(self.value))
+        self._validate_selections(self.value)
 
     def parse(self, data):
         if data == '':
@@ -78,9 +82,7 @@ class MultipleChoiceSetting(EnumStringSetting):
             return
 
         elements = data.split(',')
-        for item in elements:
-            if item not in self.choices:
-                raise ValidationException('Invalid choice: {0}'.format(item))
+        self._validate_selections(elements)
         self.value = elements
 
     def parse_form(self, data):
@@ -93,6 +95,27 @@ class MultipleChoiceSetting(EnumStringSetting):
         resp.set_cookie(name, ','.join(self.value), max_age=COOKIE_MAX_AGE)
 
 
+class SearchLanguageSetting(EnumStringSetting):
+    """Available choices may change, so user's value may not be in choices anymore"""
+
+    def parse(self, data):
+        if data not in self.choices and data != self.value:
+            # hack to give some backwards compatibility with old language cookies
+            data = str(data).replace('_', '-')
+            lang = data.split('-')[0]
+            if data in self.choices:
+                pass
+            elif lang in self.choices:
+                data = lang
+            elif data == 'nb-NO':
+                data = 'no-NO'
+            elif data == 'ar-XA':
+                data = 'ar-SA'
+            else:
+                data = self.value
+        self.value = data
+
+
 class MapSetting(Setting):
     """Setting of a value that has to be translated in order to be storable"""
 
@@ -109,7 +132,8 @@ class MapSetting(Setting):
         self.key = data
 
     def save(self, name, resp):
-        resp.set_cookie(name, bytes(self.key), max_age=COOKIE_MAX_AGE)
+        if hasattr(self, 'key'):
+            resp.set_cookie(name, self.key, max_age=COOKIE_MAX_AGE)
 
 
 class SwitchableSetting(Setting):
@@ -170,7 +194,7 @@ class EnginesSetting(SwitchableSetting):
     def _post_init(self):
         super(EnginesSetting, self)._post_init()
         transformed_choices = []
-        for engine_name, engine in self.choices.iteritems():
+        for engine_name, engine in self.choices.items():
             for category in engine.categories:
                 transformed_choice = dict()
                 transformed_choice['default_on'] = not engine.disabled
@@ -214,11 +238,12 @@ class Preferences(object):
         super(Preferences, self).__init__()
 
         self.key_value_settings = {'categories': MultipleChoiceSetting(['general'], choices=categories),
-                                   'language': EnumStringSetting('all', choices=LANGUAGE_CODES),
+                                   'language': SearchLanguageSetting(settings['search']['language'],
+                                                                     choices=LANGUAGE_CODES),
                                    'locale': EnumStringSetting(settings['ui']['default_locale'],
-                                                               choices=settings['locales'].keys()),
+                                                               choices=list(settings['locales'].keys()) + ['']),
                                    'autocomplete': EnumStringSetting(settings['search']['autocomplete'],
-                                                                     choices=autocomplete.backends.keys()),
+                                                                     choices=list(autocomplete.backends.keys()) + ['']),
                                    'image_proxy': MapSetting(settings['server']['image_proxy'],
                                                              map={'': settings['server']['image_proxy'],
                                                                   '0': False,
@@ -235,7 +260,7 @@ class Preferences(object):
         self.unknown_params = {}
 
     def parse_cookies(self, input_data):
-        for user_setting_name, user_setting in input_data.iteritems():
+        for user_setting_name, user_setting in input_data.items():
             if user_setting_name in self.key_value_settings:
                 self.key_value_settings[user_setting_name].parse(user_setting)
             elif user_setting_name == 'disabled_engines':
@@ -249,7 +274,7 @@ class Preferences(object):
         disabled_engines = []
         enabled_categories = []
         disabled_plugins = []
-        for user_setting_name, user_setting in input_data.iteritems():
+        for user_setting_name, user_setting in input_data.items():
             if user_setting_name in self.key_value_settings:
                 self.key_value_settings[user_setting_name].parse(user_setting)
             elif user_setting_name.startswith('engine_'):
@@ -270,7 +295,7 @@ class Preferences(object):
             return self.key_value_settings[user_setting_name].get_value()
 
     def save(self, resp):
-        for user_setting_name, user_setting in self.key_value_settings.iteritems():
+        for user_setting_name, user_setting in self.key_value_settings.items():
             user_setting.save(user_setting_name, resp)
         self.engines.save(resp)
         self.plugins.save(resp)

searx/query.py

@@ -21,12 +21,18 @@ from searx.languages import language_codes
 from searx.engines import (
     categories, engines, engine_shortcuts
 )
-import string
 import re
+import string
+import sys
+
+if sys.version_info[0] == 3:
+    unicode = str
+
+VALID_LANGUAGE_CODE = re.compile(r'^[a-z]{2,3}(-[a-zA-Z]{2})?$')
 
 
-class Query(object):
-    """parse query"""
+class RawTextQuery(object):
+    """parse raw text query (the value from the html input)"""
 
     def __init__(self, query, disabled_engines):
         self.query = query
@@ -66,26 +72,34 @@ class Query(object):
 
             # this force a language
             if query_part[0] == ':':
-                lang = query_part[1:].lower()
+                lang = query_part[1:].lower().replace('_', '-')
+
+                # user may set a valid, yet not selectable language
+                if VALID_LANGUAGE_CODE.match(lang):
+                    self.languages.append(lang)
+                    parse_next = True
 
                 # check if any language-code is equal with
                 # declared language-codes
                 for lc in language_codes:
-                    lang_id, lang_name, country = map(str.lower, lc)
+                    lang_id, lang_name, country, english_name = map(unicode.lower, lc)
 
                     # if correct language-code is found
                     # set it as new search-language
                     if lang == lang_id\
                        or lang_id.startswith(lang)\
                        or lang == lang_name\
-                       or lang.replace('_', ' ') == country:
+                       or lang == english_name\
+                       or lang.replace('-', ' ') == country:
                         parse_next = True
-                        self.languages.append(lang)
-                        break
+                        self.languages.append(lang_id)
+                        # to ensure best match (first match is not necessarily the best one)
+                        if lang == lang_id:
+                            break
 
             # this force a engine or category
             if query_part[0] == '!' or query_part[0] == '?':
-                prefix = query_part[1:].replace('_', ' ')
+                prefix = query_part[1:].replace('-', ' ').replace('_', ' ')
 
                 # check if prefix is equal with engine shortcut
                 if prefix in engine_shortcuts:
@@ -130,3 +144,19 @@ class Query(object):
     def getFullQuery(self):
         # get full querry including whitespaces
         return string.join(self.query_parts, '')
+
+
+class SearchQuery(object):
+    """container for all the search parameters (query, language, etc...)"""
+
+    def __init__(self, query, engines, categories, lang, safesearch, pageno, time_range):
+        self.query = query.encode('utf-8')
+        self.engines = engines
+        self.categories = categories
+        self.lang = lang
+        self.safesearch = safesearch
+        self.pageno = pageno
+        self.time_range = time_range
+
+    def __str__(self):
+        return str(self.query) + ";" + str(self.engines)

searx/results.py

@@ -1,9 +1,13 @@
 import re
+import sys
 from collections import defaultdict
 from operator import itemgetter
 from threading import RLock
-from urlparse import urlparse, unquote
 from searx.engines import engines
+from searx.url_utils import urlparse, unquote
+
+if sys.version_info[0] == 3:
+    basestring = str
 
 CONTENT_LEN_IGNORED_CHARS_REGEX = re.compile(r'[,;:!?\./\\\\ ()-_]', re.M | re.U)
 WHITESPACE_REGEX = re.compile('( |\t|\n)+', re.M | re.U)
@@ -127,16 +131,23 @@ class ResultContainer(object):
         self.infoboxes = []
         self.suggestions = set()
         self.answers = set()
+        self.corrections = set()
         self._number_of_results = []
+        self._ordered = False
+        self.paging = False
 
     def extend(self, engine_name, results):
         for result in list(results):
+            result['engine'] = engine_name
             if 'suggestion' in result:
                 self.suggestions.add(result['suggestion'])
                 results.remove(result)
             elif 'answer' in result:
                 self.answers.add(result['answer'])
                 results.remove(result)
+            elif 'correction' in result:
+                self.corrections.add(result['correction'])
+                results.remove(result)
             elif 'infobox' in result:
                 self._merge_infobox(result)
                 results.remove(result)
@@ -144,15 +155,19 @@ class ResultContainer(object):
                 self._number_of_results.append(result['number_of_results'])
                 results.remove(result)
 
-        with RLock():
-            engines[engine_name].stats['search_count'] += 1
-            engines[engine_name].stats['result_count'] += len(results)
+        if engine_name in engines:
+            with RLock():
+                engines[engine_name].stats['search_count'] += 1
+                engines[engine_name].stats['result_count'] += len(results)
 
         if not results:
             return
 
         self.results[engine_name].extend(results)
 
+        if not self.paging and engine_name in engines and engines[engine_name].paging:
+            self.paging = True
+
         for i, result in enumerate(results):
             try:
                 result['url'] = result['url'].decode('utf-8')
@@ -219,7 +234,7 @@ class ResultContainer(object):
             with RLock():
                 self._merged_results.append(result)
 
-    def get_ordered_results(self):
+    def order_results(self):
         for result in self._merged_results:
             score = result_score(result)
             result['score'] = score
@@ -235,9 +250,12 @@ class ResultContainer(object):
 
         for i, res in enumerate(results):
             # FIXME : handle more than one category per engine
-            category = engines[res['engine']].categories[0] + ':' + ''\
-                if 'template' not in res\
-                else res['template']
+            res['category'] = engines[res['engine']].categories[0]
+
+            # FIXME : handle more than one category per engine
+            category = engines[res['engine']].categories[0]\
+                + ':' + res.get('template', '')\
+                + ':' + ('img_src' if 'img_src' in res or 'thumbnail' in res else '')
 
             current = None if category not in categoryPositions\
                 else categoryPositions[category]
@@ -269,8 +287,14 @@ class ResultContainer(object):
                 # update categoryIndex
                 categoryPositions[category] = {'index': len(gresults), 'count': 8}
 
-        # return gresults
-        return gresults
+        # update _merged_results
+        self._ordered = True
+        self._merged_results = gresults
+
+    def get_ordered_results(self):
+        if not self._ordered:
+            self.order_results()
+        return self._merged_results
 
     def results_length(self):
         return len(self._merged_results)

searx/search.py

@@ -16,54 +16,155 @@ along with searx. If not, see < http://www.gnu.org/licenses/ >.
 '''
 
 import gc
+import sys
 import threading
-from thread import start_new_thread
 from time import time
 from uuid import uuid4
+import requests.exceptions
 import searx.poolrequests as requests_lib
 from searx.engines import (
     categories, engines
 )
+from searx.answerers import ask
 from searx.utils import gen_useragent
-from searx.query import Query
+from searx.query import RawTextQuery, SearchQuery, VALID_LANGUAGE_CODE
 from searx.results import ResultContainer
 from searx import logger
+from searx.plugins import plugins
+from searx.exceptions import SearxParameterException
+
+try:
+    from thread import start_new_thread
+except:
+    from _thread import start_new_thread
+
+if sys.version_info[0] == 3:
+    unicode = str
 
 logger = logger.getChild('search')
 
 number_of_searches = 0
 
 
-def search_request_wrapper(fn, url, engine_name, **kwargs):
-    ret = None
+def send_http_request(engine, request_params, start_time, timeout_limit):
+    # for page_load_time stats
+    time_before_request = time()
+
+    # create dictionary which contain all
+    # informations about the request
+    request_args = dict(
+        headers=request_params['headers'],
+        cookies=request_params['cookies'],
+        timeout=timeout_limit,
+        verify=request_params['verify']
+    )
+
+    # specific type of request (GET or POST)
+    if request_params['method'] == 'GET':
+        req = requests_lib.get
+    else:
+        req = requests_lib.post
+        request_args['data'] = request_params['data']
+
+    # send the request
+    response = req(request_params['url'], **request_args)
+
+    # is there a timeout (no parsing in this case)
+    timeout_overhead = 0.2  # seconds
+    time_after_request = time()
+    search_duration = time_after_request - start_time
+    if search_duration > timeout_limit + timeout_overhead:
+        raise requests.exceptions.Timeout(response=response)
+
+    with threading.RLock():
+        # no error : reset the suspend variables
+        engine.continuous_errors = 0
+        engine.suspend_end_time = 0
+        # update stats with current page-load-time
+        # only the HTTP request
+        engine.stats['page_load_time'] += time_after_request - time_before_request
+        engine.stats['page_load_count'] += 1
+
+    # everything is ok : return the response
+    return response
+
+
+def search_one_request(engine, query, request_params, start_time, timeout_limit):
+    # update request parameters dependent on
+    # search-engine (contained in engines folder)
+    engine.request(query, request_params)
+
+    # ignoring empty urls
+    if request_params['url'] is None:
+        return []
+
+    if not request_params['url']:
+        return []
+
+    # send request
+    response = send_http_request(engine, request_params, start_time, timeout_limit)
+
+    # parse the response
+    response.search_params = request_params
+    return engine.response(response)
+
+
+def search_one_request_safe(engine_name, query, request_params, result_container, start_time, timeout_limit):
     engine = engines[engine_name]
+
     try:
-        ret = fn(url, **kwargs)
+        # send requests and parse the results
+        search_results = search_one_request(engine, query, request_params, start_time, timeout_limit)
+
+        # add results
+        result_container.extend(engine_name, search_results)
+
+        # update engine time when there is no exception
         with threading.RLock():
-            engine.continuous_errors = 0
-            engine.suspend_end_time = 0
-    except:
-        # increase errors stats
-        with threading.RLock():
-            engine.stats['errors'] += 1
-            engine.continuous_errors += 1
-            engine.suspend_end_time = time() + min(60, engine.continuous_errors)
+            engine.stats['engine_time'] += time() - start_time
+            engine.stats['engine_time_count'] += 1
 
-        # print engine name and specific error message
-        logger.exception('engine crash: {0}'.format(engine_name))
-    return ret
+        return True
+
+    except Exception as e:
+        engine.stats['errors'] += 1
+
+        search_duration = time() - start_time
+        requests_exception = False
+
+        if (issubclass(e.__class__, requests.exceptions.Timeout)):
+            # requests timeout (connect or read)
+            logger.error("engine {0} : HTTP requests timeout"
+                         "(search duration : {1} s, timeout: {2} s) : {3}"
+                         .format(engine_name, search_duration, timeout_limit, e.__class__.__name__))
+            requests_exception = True
+        elif (issubclass(e.__class__, requests.exceptions.RequestException)):
+            # other requests exception
+            logger.exception("engine {0} : requests exception"
+                             "(search duration : {1} s, timeout: {2} s) : {3}"
+                             .format(engine_name, search_duration, timeout_limit, e))
+            requests_exception = True
+        else:
+            # others errors
+            logger.exception('engine {0} : exception : {1}'.format(engine_name, e))
+
+        # update continuous_errors / suspend_end_time
+        if requests_exception:
+            with threading.RLock():
+                engine.continuous_errors += 1
+                engine.suspend_end_time = time() + min(60, engine.continuous_errors)
+
+        #
+        return False
 
 
-def threaded_requests(requests):
-    timeout_limit = max(r[2]['timeout'] for r in requests)
-    search_start = time()
+def search_multiple_requests(requests, result_container, start_time, timeout_limit):
     search_id = uuid4().__str__()
-    for fn, url, request_args, engine_name in requests:
-        request_args['timeout'] = timeout_limit
+
+    for engine_name, query, request_params in requests:
         th = threading.Thread(
-            target=search_request_wrapper,
-            args=(fn, url, engine_name),
-            kwargs=request_args,
+            target=search_one_request_safe,
+            args=(engine_name, query, request_params, result_container, start_time, timeout_limit),
             name=search_id,
         )
         th._engine_name = engine_name
@@ -71,7 +172,7 @@ def threaded_requests(requests):
 
     for th in threading.enumerate():
         if th.name == search_id:
-            remaining_time = max(0.0, timeout_limit - (time() - search_start))
+            remaining_time = max(0.0, timeout_limit - (time() - start_time))
             th.join(remaining_time)
             if th.isAlive():
                 logger.warning('engine timeout: {0}'.format(th._engine_name))
@@ -89,175 +190,160 @@ def default_request_params():
     }
 
 
-# create a callback wrapper for the search engine results
-def make_callback(engine_name, callback, params, result_container):
+def get_search_query_from_webapp(preferences, form):
+    # no text for the query ?
+    if not form.get('q'):
+        raise SearxParameterException('q', '')
 
-    # creating a callback wrapper for the search engine results
-    def process_callback(response, **kwargs):
-        # check if redirect comparing to the True value,
-        # because resp can be a Mock object, and any attribut name returns something.
-        if response.is_redirect is True:
-            logger.debug('{0} redirect on: {1}'.format(engine_name, response))
-            return
+    # set blocked engines
+    disabled_engines = preferences.engines.get_disabled()
 
-        response.search_params = params
+    # parse query, if tags are set, which change
+    # the serch engine or search-language
+    raw_text_query = RawTextQuery(form['q'], disabled_engines)
+    raw_text_query.parse_query()
 
-        search_duration = time() - params['started']
-        # update stats with current page-load-time
-        with threading.RLock():
-            engines[engine_name].stats['page_load_time'] += search_duration
+    # set query
+    query = raw_text_query.getSearchQuery()
 
-        timeout_overhead = 0.2  # seconds
-        timeout_limit = engines[engine_name].timeout + timeout_overhead
+    # get and check page number
+    pageno_param = form.get('pageno', '1')
+    if not pageno_param.isdigit() or int(pageno_param) < 1:
+        raise SearxParameterException('pageno', pageno_param)
+    query_pageno = int(pageno_param)
 
-        if search_duration > timeout_limit:
-            with threading.RLock():
-                engines[engine_name].stats['errors'] += 1
-            return
+    # get language
+    # set specific language if set on request, query or preferences
+    # TODO support search with multible languages
+    if len(raw_text_query.languages):
+        query_lang = raw_text_query.languages[-1]
+    elif 'language' in form:
+        query_lang = form.get('language')
+    else:
+        query_lang = preferences.get_value('language')
 
-        # callback
-        search_results = callback(response)
+    # check language
+    if not VALID_LANGUAGE_CODE.match(query_lang):
+        raise SearxParameterException('language', query_lang)
 
-        # add results
-        for result in search_results:
-            result['engine'] = engine_name
+    # get safesearch
+    if 'safesearch' in form:
+        query_safesearch = form.get('safesearch')
+        # first check safesearch
+        if not query_safesearch.isdigit():
+            raise SearxParameterException('safesearch', query_safesearch)
+        query_safesearch = int(query_safesearch)
+    else:
+        query_safesearch = preferences.get_value('safesearch')
 
-        result_container.extend(engine_name, search_results)
+    # safesearch : second check
+    if query_safesearch < 0 or query_safesearch > 2:
+        raise SearxParameterException('safesearch', query_safesearch)
 
-    return process_callback
+    # get time_range
+    query_time_range = form.get('time_range')
+
+    # check time_range
+    if query_time_range not in ('None', None, '', 'day', 'week', 'month', 'year'):
+        raise SearxParameterException('time_range', query_time_range)
+
+    # query_engines
+    query_engines = raw_text_query.engines
+
+    # query_categories
+    query_categories = []
+
+    # if engines are calculated from query,
+    # set categories by using that informations
+    if query_engines and raw_text_query.specific:
+        query_categories = list(set(engine['category']
+                                    for engine in query_engines))
+
+    # otherwise, using defined categories to
+    # calculate which engines should be used
+    else:
+        # set categories/engines
+        load_default_categories = True
+        for pd_name, pd in form.items():
+            if pd_name == 'categories':
+                query_categories.extend(categ for categ in map(unicode.strip, pd.split(',')) if categ in categories)
+            elif pd_name == 'engines':
+                pd_engines = [{'category': engines[engine].categories[0],
+                               'name': engine}
+                              for engine in map(unicode.strip, pd.split(',')) if engine in engines]
+                if pd_engines:
+                    query_engines.extend(pd_engines)
+                    load_default_categories = False
+            elif pd_name.startswith('category_'):
+                category = pd_name[9:]
+
+                # if category is not found in list, skip
+                if category not in categories:
+                    continue
+
+                if pd != 'off':
+                    # add category to list
+                    query_categories.append(category)
+                elif category in query_categories:
+                    # remove category from list if property is set to 'off'
+                    query_categories.remove(category)
+
+        if not load_default_categories:
+            if not query_categories:
+                query_categories = list(set(engine['category']
+                                            for engine in query_engines))
+        else:
+            # if no category is specified for this search,
+            # using user-defined default-configuration which
+            # (is stored in cookie)
+            if not query_categories:
+                cookie_categories = preferences.get_value('categories')
+                for ccateg in cookie_categories:
+                    if ccateg in categories:
+                        query_categories.append(ccateg)
+
+            # if still no category is specified, using general
+            # as default-category
+            if not query_categories:
+                query_categories = ['general']
+
+            # using all engines for that search, which are
+            # declared under the specific categories
+            for categ in query_categories:
+                query_engines.extend({'category': categ,
+                                      'name': engine.name}
+                                     for engine in categories[categ]
+                                     if (engine.name, categ) not in disabled_engines)
+
+    return SearchQuery(query, query_engines, query_categories,
+                       query_lang, query_safesearch, query_pageno, query_time_range)
 
 
 class Search(object):
 
     """Search information container"""
 
-    def __init__(self, request):
+    def __init__(self, search_query):
         # init vars
         super(Search, self).__init__()
-        self.query = None
-        self.engines = []
-        self.categories = []
-        self.paging = False
-        self.pageno = 1
-        self.lang = 'all'
-        self.time_range = None
-        self.is_advanced = None
-
-        # set blocked engines
-        self.disabled_engines = request.preferences.engines.get_disabled()
-
+        self.search_query = search_query
         self.result_container = ResultContainer()
-        self.request_data = {}
-
-        # set specific language if set
-        self.lang = request.preferences.get_value('language')
-
-        # set request method
-        if request.method == 'POST':
-            self.request_data = request.form
-        else:
-            self.request_data = request.args
-
-        # TODO better exceptions
-        if not self.request_data.get('q'):
-            raise Exception('noquery')
-
-        # set pagenumber
-        pageno_param = self.request_data.get('pageno', '1')
-        if not pageno_param.isdigit() or int(pageno_param) < 1:
-            pageno_param = 1
-
-        self.pageno = int(pageno_param)
-
-        # parse query, if tags are set, which change
-        # the serch engine or search-language
-        query_obj = Query(self.request_data['q'], self.disabled_engines)
-        query_obj.parse_query()
-
-        # set query
-        self.query = query_obj.getSearchQuery()
-
-        # get last selected language in query, if possible
-        # TODO support search with multible languages
-        if len(query_obj.languages):
-            self.lang = query_obj.languages[-1]
-
-        self.time_range = self.request_data.get('time_range')
-        self.is_advanced = self.request_data.get('advanced_search')
-
-        self.engines = query_obj.engines
-
-        # if engines are calculated from query,
-        # set categories by using that informations
-        if self.engines and query_obj.specific:
-            self.categories = list(set(engine['category']
-                                       for engine in self.engines))
-
-        # otherwise, using defined categories to
-        # calculate which engines should be used
-        else:
-            # set categories/engines
-            load_default_categories = True
-            for pd_name, pd in self.request_data.items():
-                if pd_name == 'categories':
-                    self.categories.extend(categ for categ in map(unicode.strip, pd.split(',')) if categ in categories)
-                elif pd_name == 'engines':
-                    pd_engines = [{'category': engines[engine].categories[0],
-                                   'name': engine}
-                                  for engine in map(unicode.strip, pd.split(',')) if engine in engines]
-                    if pd_engines:
-                        self.engines.extend(pd_engines)
-                        load_default_categories = False
-                elif pd_name.startswith('category_'):
-                    category = pd_name[9:]
-
-                    # if category is not found in list, skip
-                    if category not in categories:
-                        continue
-
-                    if pd != 'off':
-                        # add category to list
-                        self.categories.append(category)
-                    elif category in self.categories:
-                        # remove category from list if property is set to 'off'
-                        self.categories.remove(category)
-
-            if not load_default_categories:
-                if not self.categories:
-                    self.categories = list(set(engine['category']
-                                               for engine in self.engines))
-                return
-
-            # if no category is specified for this search,
-            # using user-defined default-configuration which
-            # (is stored in cookie)
-            if not self.categories:
-                cookie_categories = request.preferences.get_value('categories')
-                for ccateg in cookie_categories:
-                    if ccateg in categories:
-                        self.categories.append(ccateg)
-
-            # if still no category is specified, using general
-            # as default-category
-            if not self.categories:
-                self.categories = ['general']
-
-            # using all engines for that search, which are
-            # declared under the specific categories
-            for categ in self.categories:
-                self.engines.extend({'category': categ,
-                                     'name': engine.name}
-                                    for engine in categories[categ]
-                                    if (engine.name, categ) not in self.disabled_engines)
-
-        # remove suspended engines
-        self.engines = [e for e in self.engines
-                        if engines[e['name']].suspend_end_time <= time()]
 
     # do search-request
-    def search(self, request):
+    def search(self):
         global number_of_searches
 
+        # start time
+        start_time = time()
+
+        # answeres ?
+        answerers_results = ask(self.search_query)
+
+        if answerers_results:
+            for results in answerers_results:
+                self.result_container.extend('answer', results)
+            return self.result_container
+
         # init vars
         requests = []
 
@@ -268,88 +354,79 @@ class Search(object):
         # user_agent = request.headers.get('User-Agent', '')
         user_agent = gen_useragent()
 
+        search_query = self.search_query
+
+        # max of all selected engine timeout
+        timeout_limit = 0
+
         # start search-reqest for all selected engines
-        for selected_engine in self.engines:
+        for selected_engine in search_query.engines:
             if selected_engine['name'] not in engines:
                 continue
 
             engine = engines[selected_engine['name']]
 
+            # skip suspended engines
+            if engine.suspend_end_time >= time():
+                logger.debug('Engine currently suspended: %s', selected_engine['name'])
+                continue
+
             # if paging is not supported, skip
-            if self.pageno > 1 and not engine.paging:
+            if search_query.pageno > 1 and not engine.paging:
                 continue
 
-            # if search-language is set and engine does not
-            # provide language-support, skip
-            if self.lang != 'all' and not engine.language_support:
-                continue
-
-            if self.time_range and not engine.time_range_support:
+            # if time_range is not supported, skip
+            if search_query.time_range and not engine.time_range_support:
                 continue
 
             # set default request parameters
             request_params = default_request_params()
             request_params['headers']['User-Agent'] = user_agent
             request_params['category'] = selected_engine['category']
-            request_params['started'] = time()
-            request_params['pageno'] = self.pageno
+            request_params['pageno'] = search_query.pageno
 
             if hasattr(engine, 'language') and engine.language:
                 request_params['language'] = engine.language
             else:
-                request_params['language'] = self.lang
+                request_params['language'] = search_query.lang
 
             # 0 = None, 1 = Moderate, 2 = Strict
-            request_params['safesearch'] = request.preferences.get_value('safesearch')
-            request_params['time_range'] = self.time_range
-            request_params['advanced_search'] = self.is_advanced
-
-            # update request parameters dependent on
-            # search-engine (contained in engines folder)
-            engine.request(self.query.encode('utf-8'), request_params)
-
-            if request_params['url'] is None:
-                # TODO add support of offline engines
-                pass
-
-            # create a callback wrapper for the search engine results
-            callback = make_callback(
-                selected_engine['name'],
-                engine.response,
-                request_params,
-                self.result_container)
-
-            # create dictionary which contain all
-            # informations about the request
-            request_args = dict(
-                headers=request_params['headers'],
-                hooks=dict(response=callback),
-                cookies=request_params['cookies'],
-                timeout=engine.timeout,
-                verify=request_params['verify']
-            )
-
-            # specific type of request (GET or POST)
-            if request_params['method'] == 'GET':
-                req = requests_lib.get
-            else:
-                req = requests_lib.post
-                request_args['data'] = request_params['data']
-
-            # ignoring empty urls
-            if not request_params['url']:
-                continue
+            request_params['safesearch'] = search_query.safesearch
+            request_params['time_range'] = search_query.time_range
 
             # append request to list
-            requests.append((req, request_params['url'],
-                             request_args,
-                             selected_engine['name']))
+            requests.append((selected_engine['name'], search_query.query, request_params))
 
-        if not requests:
-            return self
-        # send all search-request
-        threaded_requests(requests)
-        start_new_thread(gc.collect, tuple())
+            # update timeout_limit
+            timeout_limit = max(timeout_limit, engine.timeout)
+
+        if requests:
+            # send all search-request
+            search_multiple_requests(requests, self.result_container, start_time, timeout_limit)
+            start_new_thread(gc.collect, tuple())
 
         # return results, suggestions, answers and infoboxes
-        return self
+        return self.result_container
+
+
+class SearchWithPlugins(Search):
+
+    """Similar to the Search class but call the plugins."""
+
+    def __init__(self, search_query, ordered_plugin_list, request):
+        super(SearchWithPlugins, self).__init__(search_query)
+        self.ordered_plugin_list = ordered_plugin_list
+        self.request = request
+
+    def search(self):
+        if plugins.call(self.ordered_plugin_list, 'pre_search', self.request, self):
+            super(SearchWithPlugins, self).search()
+
+        plugins.call(self.ordered_plugin_list, 'post_search', self.request, self)
+
+        results = self.result_container.get_ordered_results()
+
+        for result in results:
+            plugins.call(self.ordered_plugin_list, 'on_result', self.request, self, result)
+
+        return self.result_container

searx/settings.yml

@@ -5,6 +5,7 @@ general:
 search:
     safe_search : 0 # Filter results. 0: None, 1: Moderate, 2: Strict
     autocomplete : "" # Existing autocomplete backends: "dbpedia", "duckduckgo", "google", "startpage", "wikipedia" - leave blank to turn it off by default
+    language : "all"
 
 server:
     port : 8888
@@ -12,12 +13,20 @@ server:
     secret_key : "ultrasecretkey" # change this!
     base_url : False # Set custom base_url. Possible values: False or "https://your.custom.host/location/"
     image_proxy : False # Proxying image results through searx
+    http_protocol_version : "1.0"  # 1.0 and 1.1 are supported
 
 ui:
-    themes_path : "" # Custom ui themes path - leave it blank if you didn't change
+    static_path : "" # Custom static path - leave it blank if you didn't change
+    templates_path : "" # Custom templates path - leave it blank if you didn't change
     default_theme : oscar # ui theme
     default_locale : "" # Default interface locale - leave blank to detect from browser information or use codes from the 'locales' config section
 
+# searx supports result proxification using an external service: https://github.com/asciimoo/morty
+# uncomment below section if you have running morty proxy
+#result_proxy:
+#    url : http://127.0.0.1:3000/
+#    key : your_morty_proxy_key
+
 outgoing: # communication with search engines
     request_timeout : 2.0 # seconds
     useragent_suffix : "" # suffix of searx_useragent, could contain informations like an email address to the administrator
@@ -84,6 +93,17 @@ engines:
     disabled : True
     shortcut : bb
 
+  - name : ccc-tv
+    engine : xpath
+    paging : False
+    search_url : https://media.ccc.de/search/?q={query}
+    url_xpath : //div[@class="caption"]/h3/a/@href
+    title_xpath : //div[@class="caption"]/h3/a/text()
+    content_xpath : //div[@class="caption"]/h4/@title
+    categories : videos
+    disabled : True
+    shortcut : c3tv
+
   - name : crossref
     engine : json_engine
     paging : True
@@ -147,6 +167,24 @@ engines:
     shortcut : ddg
     disabled : True
 
+  - name : duckduckgo images
+    engine : duckduckgo_images
+    shortcut : ddi
+    timeout: 3.0
+    disabled : True
+
+  - name : etymonline
+    engine : xpath
+    paging : True
+    search_url : http://etymonline.com/?search={query}&p={pageno}
+    url_xpath : //dt/a[1]/@href
+    title_xpath : //dt
+    content_xpath : //dd
+    suggestion_xpath : //a[@class="crossreference"]
+    first_page_num : 0
+    shortcut : et
+    disabled : True
+
 # api-key required: http://www.faroo.com/hp/api/api.html#key
 #  - name : faroo
 #    engine : faroo
@@ -177,6 +215,18 @@ engines:
 # Or you can use the html non-stable engine, activated by default
     engine : flickr_noapi
 
+  - name : free software directory
+    engine : mediawiki
+    shortcut : fsd
+    categories : it
+    base_url : https://directory.fsf.org/
+    number_of_results : 5
+# what part of a page matches the query string: title, text, nearmatch
+# title - query matches title, text - query matches the text of page, nearmatch - nearmatch in title
+    search_type : title
+    timeout : 5.0
+    disabled : True
+
   - name : frinkiac
     engine : frinkiac
     shortcut : frk
@@ -185,6 +235,7 @@ engines:
   - name : gigablast
     engine : gigablast
     shortcut : gb
+    timeout : 3.0
     disabled: True
 
   - name : gitlab
@@ -243,8 +294,8 @@ engines:
     engine        : xpath
     search_url    : https://play.google.com/store/search?q={query}&c=movies
     url_xpath     : //a[@class="title"]/@href
-    title_xpath   : //a[@class="title"]
-    content_xpath : //a[@class="subtitle"]
+    title_xpath   : //a[@class="title"]/@title
+    content_xpath : //a[contains(@class, "subtitle")]
     categories : videos
     shortcut : gpm
     disabled : True
@@ -275,8 +326,8 @@ engines:
     engine : xpath
     paging : True
     search_url : https://habrahabr.ru/search/page{pageno}/?q={query}
-    url_xpath : //div[@class="search_results"]//a[@class="post_title"]/@href
-    title_xpath : //div[@class="search_results"]//a[@class="post_title"]
+    url_xpath : //div[@class="search_results"]//a[contains(@class, "post__title_link")]/@href
+    title_xpath : //div[@class="search_results"]//a[contains(@class, "post__title_link")]
     content_xpath : //div[@class="search_results"]//div[contains(@class, "content")]
     categories : it
     timeout : 4.0
@@ -301,6 +352,33 @@ engines:
     timeout : 6.0
     disabled : True
 
+  - name: kickass
+    engine : kickass
+    shortcut : kc
+    timeout : 4.0
+    disabled : True
+
+  - name : library genesis
+    engine : xpath
+    search_url : http://libgen.io/search.php?req={query}
+    url_xpath : //a[contains(@href,"bookfi.net")]/@href
+    title_xpath : //a[contains(@href,"book/")]/text()[1]
+    content_xpath : //td/a[1][contains(@href,"=author")]/text()
+    categories : general
+    timeout : 7.0
+    disabled : True
+    shortcut : lg
+
+  - name : lobste.rs
+    engine : xpath
+    search_url : https://lobste.rs/search?utf8=%E2%9C%93&q={query}&what=stories&order=relevance
+    results_xpath : //li[contains(@class, "story")]
+    url_xpath : .//span[@class="link"]/a/@href
+    title_xpath : .//span[@class="link"]/a
+    content_xpath : .//a[@class="domain"]
+    categories : it
+    shortcut : lo
+
   - name : microsoft academic
     engine : json_engine
     paging : True
@@ -339,6 +417,13 @@ engines:
     disabled : True
     shortcut : or
 
+  - name : pdbe
+    engine : pdbe
+    shortcut : pdb
+# Hide obsolete PDB entries.
+# Default is not to hide obsolete structures
+#    hide_obsolete : False
+
   - name : photon
     engine : photon
     shortcut : ph
@@ -377,7 +462,7 @@ engines:
     timeout : 10.0
     disabled : True
 
-  - name : scanr_structures
+  - name : scanr structures
     shortcut: scs
     engine : scanr_structures
     disabled : True
@@ -399,6 +484,30 @@ engines:
     shortcut : scc
     disabled : True
 
+  - name : framalibre
+    engine : framalibre
+    shortcut : frl
+    disabled : True
+
+#  - name : searx
+#    engine : searx_engine
+#    shortcut : se
+#    instance_urls :
+#        - http://127.0.0.1:8888/
+#        - ...
+#    disabled : True
+
+  - name : semantic scholar
+    engine : xpath
+    paging : True
+    search_url : https://www.semanticscholar.org/search?q={query}&sort=relevance&page={pageno}&ae=false
+    results_xpath : //article
+    url_xpath : .//div[@class="search-result-title"]/a/@href
+    title_xpath : .//div[@class="search-result-title"]/a
+    content_xpath : .//div[@class="search-result-abstract"]
+    shortcut : se
+    categories : science
+
   - name : spotify
     engine : spotify
     shortcut : stf
@@ -418,8 +527,8 @@ engines:
 
   - name : ixquick
     engine : startpage
-    base_url : 'https://www.ixquick.com/'
-    search_url : 'https://www.ixquick.com/do/search'
+    base_url : 'https://www.ixquick.eu/'
+    search_url : 'https://www.ixquick.eu/do/search'
     shortcut : iq
     timeout : 6.0
     disabled : True
@@ -450,9 +559,9 @@ engines:
 #  - name : urbandictionary
 #    engine        : xpath
 #    search_url    : http://www.urbandictionary.com/define.php?term={query}
-#    url_xpath     : //div[@class="word"]//a/@href
-#    title_xpath   : //div[@class="word"]//a
-#    content_xpath : //div[@class="definition"]
+#    url_xpath     : //*[@class="word"]/@href
+#    title_xpath   : //*[@class="def-header"]
+#    content_xpath : //*[@class="meaning"]
 #    shortcut : ud
 
   - name : yahoo
@@ -495,6 +604,12 @@ engines:
     timeout: 6.0
     categories : science
 
+  - name : seedpeer
+    engine : seedpeer
+    shortcut: speu
+    categories: files, music, videos
+    disabled: True
+
   - name : dictzone
     engine : dictzone
     shortcut : dc
@@ -508,6 +623,22 @@ engines:
     # See : http://mymemory.translated.net/doc/usagelimits.php
     # api_key : ''
 
+  - name : voat
+    engine: xpath
+    shortcut: vo
+    categories: social media
+    search_url : https://voat.co/search?q={query}
+    url_xpath : //p[contains(@class, "title")]/a/@href
+    title_xpath : //p[contains(@class, "title")]/a
+    content_xpath : //span[@class="domain"]
+    timeout : 10.0
+    disabled : True
+
+  - name : 1337x
+    engine : 1337x
+    shortcut : 1337x
+    disabled : True
+
 #The blekko technology and team have joined IBM Watson! -> https://blekko.com/
 #  - name : blekko images
 #    engine : blekko_images
@@ -531,10 +662,13 @@ engines:
 locales:
     en : English
     bg : Български (Bulgarian)
+    cs : Čeština (Czech)
     de : Deutsch (German)
+    de_DE : Deutsch (German_Germany)
     el_GR : Ελληνικά (Greek_Greece)
     eo : Esperanto (Esperanto)
     es : Español (Spanish)
+    fi : Suomi (Finnish)
     fr : Français (French)
     he : עברית (Hebrew)
     hu : Magyar (Hungarian)
@@ -545,5 +679,8 @@ locales:
     pt_BR : Português (Portuguese_Brazil)
     ro : Română (Romanian)
     ru : Русский (Russian)
+    sk : Slovenčina (Slovak)
+    sv : Svenska (Swedish)
     tr : Türkçe (Turkish)
+    uk : українська мова (Ukrainian)
     zh : 中文 (Chinese)

searx/settings_robot.yml

@@ -5,6 +5,7 @@ general:
 search:
     safe_search : 0
     autocomplete : ""
+    language: "all"
 
 server:
     port : 11111
@@ -12,10 +13,12 @@ server:
     secret_key : "ultrasecretkey" # change this!
     base_url : False
     image_proxy : False
+    http_protocol_version : "1.0"
 
 ui:
-    themes_path : ""
-    default_theme : default
+    static_path : ""
+    templates_path : ""
+    default_theme : oscar
     default_locale : ""
 
 outgoing:
@@ -23,12 +26,12 @@ outgoing:
     useragent_suffix : ""
 
 engines:
-  - name : general_dummy
+  - name : general dummy
     engine : dummy
     categories : general
     shortcut : gd
 
-  - name : dummy_dummy
+  - name : dummy dummy
     engine : dummy
     categories : dummy
     shortcut : dd

searx/static/plugins/js/infinite_scroll.js

@@ -5,7 +5,7 @@ $(document).ready(function() {
             var formData = $('#pagination form:last').serialize();
             if (formData) {
                 $('#pagination').html('<div class="loading-spinner"></div>');
-                $.post('/', formData, function (data) {
+                $.post('./', formData, function (data) {
                     var body = $(data);
                     $('#pagination').remove();
                     $('#main_results').append('<hr/>');