src/typosaurus_sandbox/research/frontier.py

@@ -104,6 +104,7 @@ class QueryFrontier:
         self._lock = threading.Lock()
         self._seen_queries: set[str] = set()
         self._seen_urls: set[str] = set()
+        self._seen_url_order: list[str] = []
         self._seen_content: set[str] = set()
         self._origins: dict[str, str] = {}
         self._pending: asyncio.Queue[str] = asyncio.Queue()
@@ -224,3 +225,4 @@ class QueryFrontier:
                 content_duplicates_skipped=self._content_duplicates_skipped,
             )
 
+

tests/test_research_integration.py

@@ -0,0 +1,70 @@
+# retoor <retoor@molodetz.nl>
+
+import asyncio
+import unittest
+import urllib.request
+from typing import Any, AsyncIterator
+from unittest import mock
+
+from typosaurus_sandbox.research.client import RsearchClient
+from typosaurus_sandbox.research.config import ResearchConfig
+from typosaurus_sandbox.research.frontier import QueryFrontier
+from typosaurus_sandbox.research.pipeline import PipelineReport, ResearchPipeline, WorkItem
+
+PROBE_SUBJECT = "python asyncio"
+RSEARCH_BASE_URL = "https://rsearch.app.molodetz.nl"
+RUN_TIMEOUT_SECONDS = 60.0
+
+
+class TestLiveResearchProbe(unittest.TestCase):
+
+    def test_bounded_probe_runs_against_live_rsearch_api(self) -> None:
+        config = ResearchConfig(
+            base_url=RSEARCH_BASE_URL,
+            max_concurrency=2,
+            default_count=2,
+            request_timeout_seconds=30.0,
+        )
+        self.assertEqual(config.base_url, RSEARCH_BASE_URL)
+        client = RsearchClient(config)
+        frontier = QueryFrontier(PROBE_SUBJECT)
+        requested: list[str] = []
+
+        original_urlopen = urllib.request.urlopen
+
+        def recording_urlopen(request: urllib.request.Request, timeout: float | None = None) -> Any:
+            requested.append(request.get_full_url())
+            return original_urlopen(request, timeout=timeout)
+
+        with mock.patch("urllib.request.urlopen", side_effect=recording_urlopen):
+            first = asyncio.run(self._bounded_run(client, frontier))
+            first_request_count = len(requested)
+            second = asyncio.run(self._bounded_run(client, frontier))
+            second_request_count = len(requested)
+
+        self.assertGreaterEqual(first.requests_succeeded, 1)
+        self.assertGreaterEqual(first.urls_found, 1)
+        self.assertGreaterEqual(first.contents_seen, 1)
+        self.assertFalse(any(outcome.cache_hit for outcome in first.outcomes))
+        stats = frontier.snapshot()
+        self.assertGreaterEqual(stats.urls_seen, 1)
+        self.assertGreaterEqual(stats.content_seen, 1)
+        self.assertGreaterEqual(first_request_count, 1)
+        for url in requested:
+            self.assertTrue(url.startswith(RSEARCH_BASE_URL), url)
+        self.assertTrue(any("/search" in url for url in requested))
+        self.assertEqual(second.requests_succeeded, 1)
+        self.assertTrue(any(outcome.cache_hit for outcome in second.outcomes))
+        self.assertEqual(second_request_count, first_request_count)
+
+    async def _bounded_run(self, client: RsearchClient, frontier: QueryFrontier) -> PipelineReport:
+        async def items() -> AsyncIterator[WorkItem]:
+            yield WorkItem("web", PROBE_SUBJECT)
+
+        pipeline = ResearchPipeline(client, frontier)
+        return await asyncio.wait_for(pipeline.run(items()), timeout=RUN_TIMEOUT_SECONDS)
+
+
+if __name__ == "__main__":
+    unittest.main()
+