[enh] request arg refactor

Browse Source
This commit is contained in:
asciimoo 2013-11-03 00:40:45 +01:00
parent 443af2ba83
commit d2a952ecdf
1 changed files with 18 additions and 14 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

32
searx/engines/__init__.py
View File

@ -104,36 +104,40 @@ def search(query, request, selected_categories):
selected_engines = [] selected_engines = []
number_of_searches += 1 number_of_searches += 1
user_agent = request.headers.get('User-Agent', '') user_agent = request.headers.get('User-Agent', '')
if not len(selected_categories): if not len(selected_categories):
selected_categories = ['general'] selected_categories = ['general']
for categ in selected_categories: for categ in selected_categories:
selected_engines.extend({'category': categ, 'name': x.name} for x in categories[categ]) selected_engines.extend({'category': categ, 'name': x.name} for x in categories[categ])
for selected_engine in selected_engines: for selected_engine in selected_engines:
if selected_engine['name'] not in engines: if selected_engine['name'] not in engines:
continue continue
engine = engines[selected_engine['name']] engine = engines[selected_engine['name']]
request_params = default_request_params() request_params = default_request_params()
request_params['headers']['User-Agent'] = user_agent request_params['headers']['User-Agent'] = user_agent
request_params['category'] = selected_engine['category'] request_params['category'] = selected_engine['category']
request_params['started'] = datetime.now() request_params['started'] = datetime.now()
request_params = engine.request(query, request_params) request_params = engine.request(query, request_params)
callback = make_callback(selected_engine['name'], results, engine.response, request_params) callback = make_callback(selected_engine['name'], results, engine.response, request_params)
request_args = dict(headers = request_params['headers']
,hooks = dict(response=callback)
,cookies = request_params['cookies']
,timeout = settings.request_timeout
)
if request_params['method'] == 'GET': if request_params['method'] == 'GET':
req = grequests.get(request_params['url'] req = grequests.get
,headers = request_params['headers']
,hooks = dict(response=callback)
,cookies = request_params['cookies']
,timeout = settings.request_timeout
)
else: else:
req = grequests.post(request_params['url'] req = grequests.post
,data = request_params['data'] request_args['data'] = request_params['data']
,headers = request_params['headers']
,hooks = dict(response=callback) requests.append(req(request_params['url'], **request_args))
,cookies = request_params['cookies']
,timeout = settings.request_timeout
)
requests.append(req)
grequests.map(requests) grequests.map(requests)
for engine_name,engine_results in results.items(): for engine_name,engine_results in results.items():
engines[engine_name].stats['search_count'] += 1 engines[engine_name].stats['search_count'] += 1