|
# retoor <retoor@molodetz.nl>
|
|
|
|
import asyncio
|
|
import unittest
|
|
import urllib.request
|
|
from typing import Any, AsyncIterator
|
|
from unittest import mock
|
|
|
|
from typosaurus_sandbox.research.client import RsearchClient
|
|
from typosaurus_sandbox.research.config import ResearchConfig
|
|
from typosaurus_sandbox.research.frontier import QueryFrontier
|
|
from typosaurus_sandbox.research.pipeline import PipelineReport, ResearchPipeline, WorkItem
|
|
|
|
PROBE_SUBJECT = "python asyncio"
|
|
RSEARCH_BASE_URL = "https://rsearch.app.molodetz.nl"
|
|
RUN_TIMEOUT_SECONDS = 60.0
|
|
|
|
|
|
class TestLiveResearchProbe(unittest.TestCase):
|
|
|
|
def test_bounded_probe_runs_against_live_rsearch_api(self) -> None:
|
|
config = ResearchConfig(
|
|
base_url=RSEARCH_BASE_URL,
|
|
max_concurrency=2,
|
|
default_count=2,
|
|
request_timeout_seconds=30.0,
|
|
)
|
|
self.assertEqual(config.base_url, RSEARCH_BASE_URL)
|
|
client = RsearchClient(config)
|
|
frontier = QueryFrontier(PROBE_SUBJECT)
|
|
requested: list[str] = []
|
|
|
|
original_urlopen = urllib.request.urlopen
|
|
|
|
def recording_urlopen(request: urllib.request.Request, timeout: float | None = None) -> Any:
|
|
requested.append(request.get_full_url())
|
|
return original_urlopen(request, timeout=timeout)
|
|
|
|
with mock.patch("urllib.request.urlopen", side_effect=recording_urlopen):
|
|
first = asyncio.run(self._bounded_run(client, frontier))
|
|
first_request_count = len(requested)
|
|
second = asyncio.run(self._bounded_run(client, frontier))
|
|
second_request_count = len(requested)
|
|
|
|
self.assertGreaterEqual(first.requests_succeeded, 1)
|
|
self.assertGreaterEqual(first.urls_found, 1)
|
|
self.assertGreaterEqual(first.contents_seen, 1)
|
|
self.assertFalse(any(outcome.cache_hit for outcome in first.outcomes))
|
|
stats = frontier.snapshot()
|
|
self.assertGreaterEqual(stats.urls_seen, 1)
|
|
self.assertGreaterEqual(stats.content_seen, 1)
|
|
self.assertGreaterEqual(first_request_count, 1)
|
|
for url in requested:
|
|
self.assertTrue(url.startswith(RSEARCH_BASE_URL), url)
|
|
self.assertTrue(any("/search" in url for url in requested))
|
|
self.assertEqual(second.requests_succeeded, 1)
|
|
self.assertTrue(any(outcome.cache_hit for outcome in second.outcomes))
|
|
self.assertEqual(second_request_count, first_request_count)
|
|
|
|
async def _bounded_run(self, client: RsearchClient, frontier: QueryFrontier) -> PipelineReport:
|
|
async def items() -> AsyncIterator[WorkItem]:
|
|
yield WorkItem("web", PROBE_SUBJECT)
|
|
|
|
pipeline = ResearchPipeline(client, frontier)
|
|
return await asyncio.wait_for(pipeline.run(items()), timeout=RUN_TIMEOUT_SECONDS)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|
|
|