# SPDX-License-Identifier: AGPL-3.0-or-later
# pylint: disable=missing-module-docstring
from collections import defaultdict
import mock
from searx.engines import json_engine
from searx import logger
from tests import SearxTestCase
logger = logger.getChild('engines')
class TestJsonEngine(SearxTestCase): # pylint: disable=missing-class-docstring
json = """
[
{
"title": "title0",
"content": "content0",
"url": "https://example.com/url0",
"images": [
{
"thumb": "https://example.com/thumb00"
},
{
"thumb": "https://example.com/thumb01"
}
]
},
{
"title": "<h1>title1</h1>",
"content": "<h2>content1</h2>",
"url": "https://example.com/url1",
"images": [
{
"thumb": "https://example.com/thumb10"
},
{
"thumb": "https://example.com/thumb11"
}
]
},
{
"title": "title2",
"content": "content2",
"url": 2,
"images": [
{
"thumb": "thumb20"
},
{
"thumb": 21
}
]
}
]
"""
json_result_query = """
{
"data": {
"results": [
{
"title": "title0",
"content": "content0",
"url": "https://example.com/url0",
"images": [
{
"thumb": "https://example.com/thumb00"
},
{
"thumb": "https://example.com/thumb01"
}
]
},
{
"title": "<h1>title1</h1>",
"content": "<h2>content1</h2>",
"url": "https://example.com/url1",
"images": [
{
"thumb": "https://example.com/thumb10"
},
{
"thumb": "https://example.com/thumb11"
}
]
},
{
"title": "title2",
"content": "content2",
"url": 2,
"images": [
{
"thumb": "thumb20"
},
{
"thumb": 21
}
]
}
],
"suggestions": [
"suggestion0",
"suggestion1"
]
}
}
"""
def setUp(self):
json_engine.logger = logger.getChild('test_json_engine')
def test_request(self):
json_engine.search_url = 'https://example.com/{query}'
json_engine.categories = []
json_engine.paging = False
query = 'test_query'
dicto = defaultdict(dict)
dicto['language'] = 'all'
dicto['pageno'] = 1
params = json_engine.request(query, dicto)
self.assertIn('url', params)
self.assertEqual('https://example.com/test_query', params['url'])
json_engine.search_url = 'https://example.com/q={query}&p={pageno}'
json_engine.paging = True
query = 'test_query'
dicto = defaultdict(dict)
dicto['language'] = 'all'
dicto['pageno'] = 1
params = json_engine.request(query, dicto)
self.assertIn('url', params)
self.assertEqual('https://example.com/q=test_query&p=1', params['url'])
json_engine.search_url = 'https://example.com/'
json_engine.paging = True
json_engine.request_body = '{{"page": {pageno}, "query": "{query}"}}'
query = 'test_query'
dicto = defaultdict(dict)
dicto['language'] = 'all'
dicto['pageno'] = 1
params = json_engine.request(query, dicto)
self.assertIn('data', params)
self.assertEqual('{"page": 1, "query": "test_query"}', params['data'])
def test_response(self):
# without results_query
json_engine.results_query = ''
json_engine.url_query = 'url'
json_engine.url_prefix = ''
json_engine.title_query = 'title'
json_engine.content_query = 'content'
json_engine.thumbnail_query = 'images/thumb'
json_engine.thumbnail_prefix = ''
json_engine.title_html_to_text = False
json_engine.content_html_to_text = False
json_engine.categories = []
self.assertRaises(AttributeError, json_engine.response, None)
self.assertRaises(AttributeError, json_engine.response, [])
self.assertRaises(AttributeError, json_engine.response, '')
self.assertRaises(AttributeError, json_engine.response, '[]')
response = mock.Mock(text='{}', status_code=200)
self.assertEqual(json_engine.response(response), [])
response = mock.Mock(text=self.json, status_code=200)
results = json_engine.response(response)
self.assertEqual(type(results), list)
self.assertEqual(len(results), 3)
self.assertEqual(results[0]['title'], 'title0')
self.assertEqual(results[0]['url'], 'https://example.com/url0')
self.assertEqual(results[0]['content'], 'content0')
self.assertEqual(results[0]['thumbnail'], 'https://example.com/thumb00')
self.assertEqual(results[1]['title'], '<h1>title1</h1>')
self.assertEqual(results[1]['url'], 'https://example.com/url1')
self.assertEqual(results[1]['content'], '<h2>content1</h2>')
self.assertEqual(results[1]['thumbnail'], 'https://example.com/thumb10')
# with prefix and suggestions without results_query
json_engine.url_prefix = 'https://example.com/url'
json_engine.thumbnail_query = 'images/1/thumb'
json_engine.thumbnail_prefix = 'https://example.com/thumb'
results = json_engine.response(response)
self.assertEqual(type(results), list)
self.assertEqual(len(results), 3)
self.assertEqual(results[2]['title'], 'title2')
self.assertEqual(results[2]['url'], 'https://example.com/url2')
self.assertEqual(results[2]['content'], 'content2')
self.assertEqual(results[2]['thumbnail'], 'https://example.com/thumb21')
self.assertFalse(results[0].get('is_onion', False))
# results are onion urls without results_query
json_engine.categories = ['onions']
results = json_engine.response(response)
self.assertTrue(results[0]['is_onion'])
def test_response_results_json(self):
# with results_query
json_engine.results_query = 'data/results'
json_engine.url_query = 'url'
json_engine.url_prefix = ''
json_engine.title_query = 'title'
json_engine.content_query = 'content'
json_engine.thumbnail_query = 'images/1/thumb'
json_engine.thumbnail_prefix = ''
json_engine.title_html_to_text = True
json_engine.content_html_to_text = True
json_engine.categories = []
self.assertRaises(AttributeError, json_engine.response, None)
self.assertRaises(AttributeError, json_engine.response, [])
self.assertRaises(AttributeError, json_engine.response, '')
self.assertRaises(AttributeError, json_engine.response, '[]')
response = mock.Mock(text='{}', status_code=200)
self.assertEqual(json_engine.response(response), [])
response = mock.Mock(text=self.json_result_query, status_code=200)
results = json_engine.response(response)
self.assertEqual(type(results), list)
self.assertEqual(len(results), 3)
self.assertEqual(results[0]['title'], 'title0')
self.assertEqual(results[0]['url'], 'https://example.com/url0')
self.assertEqual(results[0]['content'], 'content0')
self.assertEqual(results[0]['thumbnail'], 'https://example.com/thumb01')
self.assertEqual(results[1]['title'], 'title1')
self.assertEqual(results[1]['url'], 'https://example.com/url1')
self.assertEqual(results[1]['content'], 'content1')
self.assertEqual(results[1]['thumbnail'], 'https://example.com/thumb11')
# with prefix and suggestions with results_query
json_engine.url_prefix = 'https://example.com/url'
json_engine.thumbnail_query = 'images/1/thumb'
json_engine.thumbnail_prefix = 'https://example.com/thumb'
json_engine.suggestion_query = 'data/suggestions'
results = json_engine.response(response)
self.assertEqual(type(results), list)
self.assertEqual(len(results), 4)
self.assertEqual(results[2]['title'], 'title2')
self.assertEqual(results[2]['url'], 'https://example.com/url2')
self.assertEqual(results[2]['content'], 'content2')
self.assertEqual(results[2]['thumbnail'], 'https://example.com/thumb21')
self.assertEqual(results[3]['suggestion'], ['suggestion0', 'suggestion1'])
self.assertFalse(results[0].get('is_onion', False))
# results are onion urls with results_query
json_engine.categories = ['onions']
results = json_engine.response(response)
self.assertTrue(results[0]['is_onion'])
Reference in New Issue View Git Blame Copy Permalink