typosaurus/typosaurus-sandbox
1
0
Fork 0
Code Issues 1 Pull Requests 1 Actions Packages Projects Releases Wiki Activity

feat(tanya): Audit tests/ for skipped, disabled, or weakened tests

Browse Source 
Outcome: done
Changed: none
Verified by: PYTHONPATH=src python3 -m unittest discover -s tests -q -> "Ran 226 tests in 0.634s OK", EXIT_CODE=0
Findings: Criterion 1 PASS - grep for unittest.skip|skipIf|skipUnless|SkipTest|expectedFailure|pytest.mark.skip|pytest.skip|xfail|@skip|@disabled|pytestmark across tests/ returned 0 hits; case-insensitive skipif|skipunless|onlyif|not implemented also 0; runtime report shows no skipped/expected-failure suffix
Findings: Criterion 2 PASS - grep '^\s*(pass|\.\.\.)\s*$' returned 0 hits; AST scan of all 226 test_* functions found none with only-pass body and every one contains >=1 assertion (bare assert or self.assert*/fail* call)
Findings: Criterion 3 PASS - all 25 broad 'skip' grep hits individually inspected and are duplicates_skipped/cache counters or test names, not directives: tests/test_research_engine.py:132-136, tests/test_research_dedup.py:105-233, tests/test_research_scheduling.py:226-496, tests/test_research_client.py:426,434; bare 'return' at tests/test_research_scheduling.py:149,175,289 are worker loop-exit control flow (assertions at 157-162,183-186,298-300); tests/test_research_pipeline.py:333 tests exception handling with assertions at 344-351
Findings: Criterion 4 PASS - evidence recorded as file:line references above and stored in tree finding
Findings: Supplemental sweep for __test__|no cover|pragma|.skip(|mark. returned 0 hits; suite has grown to 226 tests (previous run 219) with no skipped/expected failures reported
Open

Typosaurus-Run: 4e2afb673c7f4578a12276d9181b982d
Typosaurus-Node: 46ed07b2395240b297e0fedbe3b672cd
Typosaurus-Agent: @tanya
Refs: #31
This commit is contained in:
typosaurus
2026-08-08 02:54:49 +00:00
parent 808d6b4f83
commit 472a59e122
4 changed files with 291 additions and 33 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/test_research_engine.py
+123 -10
View File
@@ -2,12 +2,18 @@
import unittest
from collections.abc import Callable
from unittest.mock import patch
from typosaurus_sandbox.research.client import RsearchError
from typosaurus_sandbox.research.config import ResearchConfig
from typosaurus_sandbox.research.engine import ResearchEngine
from typosaurus_sandbox.research.envelopes import ChatResponse, DescribeResponse, SearchResponse, SearchResult
async def _no_sleep(delay: float) -> None:
return None
class FakeResearchClient:
def __init__(
self,
@@ -22,7 +28,7 @@ class FakeResearchClient:
self._web_result_factory = web_result_factory
self._chat_text = chat_text
self._describe_text = describe_text
self.calls: list[tuple[str, str, str | None]] = []
self.calls: list[tuple[str, str, str | None, bool, bool]] = []
def search_cached(
self,
@@ -53,18 +59,18 @@ class FakeResearchClient:
ai: bool = False,
cache: bool = True,
) -> SearchResponse:
self.calls.append(("search", query, type))
self.calls.append(("search", query, type, deep, ai))
if type == "images":
return SearchResponse(query=query, source="wikimedia", count=0, success=True, results=[])
results = self._web_result_factory(query) if self._web_result_factory is not None else list(self._web_results)
return SearchResponse(query=query, source="duckduckgo", count=len(results), success=True, results=results)
async def chat(self, prompt: str, *, system: str | None = None, json_mode: bool = False, cache: bool = True) -> ChatResponse:
self.calls.append(("chat", prompt, None))
self.calls.append(("chat", prompt, None, False, False))
return ChatResponse(response=self._chat_text, prompt=prompt)
async def describe(self, url: str) -> DescribeResponse:
self.calls.append(("describe", url, None))
self.calls.append(("describe", url, None, False, False))
return DescribeResponse(description=self._describe_text, url=url)
@@ -92,7 +98,7 @@ class TestEngineClosureDetection(unittest.IsolatedAsyncioTestCase):
self.assertEqual(report.contents_seen, 0)
self.assertEqual(report.content_types, {"web": 1, "images": 1, "chat": 1})
self.assertEqual(len(client.calls), 3)
self.assertEqual({kind for kind, _, _ in client.calls}, {"search", "chat"})
self.assertEqual({call[0] for call in client.calls}, {"search", "chat"})
async def test_run_discovery_rounds_then_closes(self) -> None:
client = FakeResearchClient(
@@ -134,11 +140,11 @@ class TestEngineClosureDetection(unittest.IsolatedAsyncioTestCase):
self.assertEqual(report.cache_misses, 10)
self.assertEqual(report.content_types, {"web": 3, "images": 3, "chat": 3, "describe": 1})
self.assertEqual(len(client.calls), 10)
self.assertEqual(sum(1 for kind, _, type_value in client.calls if kind == "search" and type_value is None), 3)
self.assertEqual(sum(1 for kind, _, type_value in client.calls if kind == "search" and type_value == "images"), 3)
self.assertEqual(sum(1 for kind, _, _ in client.calls if kind == "chat"), 3)
self.assertEqual(sum(1 for kind, value, _ in client.calls if kind == "describe"), 1)
self.assertIn(("describe", "https://example.com/alpha", None), client.calls)
self.assertEqual(sum(1 for kind, _, type_value, _, _ in client.calls if kind == "search" and type_value is None), 3)
self.assertEqual(sum(1 for kind, _, type_value, _, _ in client.calls if kind == "search" and type_value == "images"), 3)
self.assertEqual(sum(1 for kind, _, _, _, _ in client.calls if kind == "chat"), 3)
self.assertEqual(sum(1 for kind, _, _, _, _ in client.calls if kind == "describe"), 1)
self.assertIn(("describe", "https://example.com/alpha", None, False, False), client.calls)
async def test_new_content_alone_does_not_prevent_closure(self) -> None:
def factory(query: str) -> list[SearchResult]:
@@ -180,6 +186,107 @@ class TestEngineClosureDetection(unittest.IsolatedAsyncioTestCase):
self.assertTrue(report_dict["closed"])
class FailingWebClient(FakeResearchClient):
def __init__(
self,
*,
web_results: list[SearchResult] | None = None,
web_result_factory: Callable[[str], list[SearchResult]] | None = None,
chat_text: str = "",
describe_text: str = "",
failures_before_success: int = 0,
) -> None:
super().__init__(
web_results=web_results,
web_result_factory=web_result_factory,
chat_text=chat_text,
describe_text=describe_text,
)
self._web_failures_left = failures_before_success
async def search(
self,
query: str,
*,
source: str | None = None,
count: int | None = None,
content: bool = False,
type: str | None = None,
deep: bool = False,
ai: bool = False,
cache: bool = True,
) -> SearchResponse:
self.calls.append(("search", query, type, deep, ai))
if type == "images":
return SearchResponse(query=query, source="wikimedia", count=0, success=True, results=[])
if self._web_failures_left > 0:
self._web_failures_left -= 1
raise RsearchError("search failed", 503)
results = self._web_result_factory(query) if self._web_result_factory is not None else list(self._web_results)
return SearchResponse(query=query, source="duckduckgo", count=len(results), success=True, results=results)
class TestEngineDeepAiWiring(unittest.IsolatedAsyncioTestCase):
async def test_web_search_work_items_issue_deep_and_ai_for_seed_and_subtopics(self) -> None:
def factory(query: str) -> list[SearchResult]:
return [
SearchResult(
title="subtopic alpha",
url="https://example.com/subtopic",
description="subtopic details",
content="subtopic body",
)
]
client = FakeResearchClient(web_result_factory=factory, chat_text="", describe_text="")
engine = ResearchEngine(client=client)
report = await engine.run("seed topic")
self.assertTrue(report.closed)
web_calls = [call for call in client.calls if call[0] == "search" and call[2] is None]
self.assertEqual(len(web_calls), 3)
self.assertEqual({call[1] for call in web_calls}, {"seed topic", "subtopic alpha", "subtopic details"})
self.assertTrue(all(call[3] and call[4] for call in web_calls))
class TestEngineClosureOnFailures(unittest.IsolatedAsyncioTestCase):
async def test_round_and_report_not_closed_when_request_failed(self) -> None:
client = FailingWebClient(failures_before_success=100)
engine = ResearchEngine(client=client)
with patch("typosaurus_sandbox.research.pipeline.asyncio.sleep", side_effect=_no_sleep):
report = await engine.run("subject")
self.assertFalse(report.closed)
self.assertEqual(report.requests_failed, 1)
self.assertEqual(report.requests_succeeded, 2)
self.assertEqual(report.total_rounds, 1)
first = report.rounds[0]
self.assertEqual(first.requests_failed, 1)
self.assertEqual(first.requests_succeeded, 2)
self.assertEqual(first.new_urls, 0)
self.assertEqual(first.new_queries, 0)
self.assertFalse(first.closed)
async def test_later_closed_round_keeps_report_unclosed_after_earlier_failure(self) -> None:
client = FailingWebClient(
chat_text="Reference at https://chat.example.org/note",
describe_text="",
failures_before_success=3,
)
engine = ResearchEngine(client=client)
with patch("typosaurus_sandbox.research.pipeline.asyncio.sleep", side_effect=_no_sleep):
report = await engine.run("subject")
self.assertEqual(report.total_rounds, 2)
first = report.rounds[0]
self.assertEqual(first.requests_failed, 1)
self.assertFalse(first.closed)
second = report.rounds[1]
self.assertEqual(second.requests_failed, 0)
self.assertTrue(second.closed)
self.assertEqual(report.requests_failed, 1)
self.assertFalse(report.closed)
class TestEngineInputValidation(unittest.IsolatedAsyncioTestCase):
async def test_empty_subject_raises_without_requests(self) -> None:
@@ -196,3 +303,9 @@ if __name__ == "__main__":
tests/test_research_pipeline.py
+88 -3
View File
@@ -2,12 +2,14 @@
import unittest
from typing import Any, AsyncIterator
from unittest.mock import patch
from typosaurus_sandbox.research.client import RsearchError
from typosaurus_sandbox.research.config import ResearchConfig
from typosaurus_sandbox.research.envelopes import ChatResponse, DeepReport, DescribeResponse, SearchResponse, SearchResult
from typosaurus_sandbox.research.frontier import QueryFrontier
from typosaurus_sandbox.research.pipeline import (
RETRY_MAX_ATTEMPTS,
Extraction,
ResearchPipeline,
WorkItem,
@@ -16,15 +18,31 @@ from typosaurus_sandbox.research.pipeline import (
)
async def _no_sleep(delay: float) -> None:
return None
class StubResearchClient:
def __init__(self) -> None:
self.config = ResearchConfig(max_concurrency=4, default_count=5)
self.cache_hits: dict[tuple[str, str], Any] = {}
self.error_on: set[tuple[str, str]] = set()
self.explode_on: set[tuple[str, str]] = set()
self.failures_remaining: dict[tuple[str, str], int] = {}
self.chat_response = ChatResponse(response="chat answer")
self.describe_response = DescribeResponse(description="described page")
self.calls: list[tuple[str, str]] = []
def _maybe_fail(self, key: tuple[str, str], error: RsearchError) -> None:
if key in self.explode_on:
raise ValueError("unexpected boom")
if key in self.error_on:
raise error
remaining = self.failures_remaining.get(key, 0)
if remaining > 0:
self.failures_remaining[key] = remaining - 1
raise error
def search_cached(
self,
query: str,
@@ -57,8 +75,7 @@ class StubResearchClient:
cache: bool = True,
) -> SearchResponse:
self.calls.append(("search", query))
if ("search", query) in self.error_on:
raise RsearchError("search failed", 503)
self._maybe_fail(("search", query), RsearchError("search failed", 503))
return SearchResponse(
query=query,
source="duckduckgo",
@@ -242,7 +259,8 @@ class TestPipelineOutcomes(unittest.IsolatedAsyncioTestCase):
yield WorkItem("web", "good")
pipeline = ResearchPipeline(client, frontier)
report = await pipeline.run(items())
with patch("typosaurus_sandbox.research.pipeline.asyncio.sleep", side_effect=_no_sleep):
report = await pipeline.run(items())
self.assertEqual(report.requests_succeeded, 1)
self.assertEqual(report.requests_failed, 1)
self.assertEqual(len(report.outcomes), 2)
@@ -271,6 +289,73 @@ class TestPipelineOutcomes(unittest.IsolatedAsyncioTestCase):
self.assertEqual(endpoints, {"web": "/search", "images": "/search", "describe": "/describe", "chat": "/chat"})
class TestPipelineRetryAndFailureAccounting(unittest.IsolatedAsyncioTestCase):
async def test_transient_failure_retried_with_backoff_then_succeeds(self) -> None:
client = StubResearchClient()
client.failures_remaining[("search", "flaky")] = 2
frontier = QueryFrontier()
pipeline = ResearchPipeline(client, frontier)
delays: list[float] = []
async def fake_sleep(delay: float) -> None:
delays.append(delay)
with patch("typosaurus_sandbox.research.pipeline.asyncio.sleep", side_effect=fake_sleep):
outcome = await pipeline.process(WorkItem("web", "flaky"))
self.assertTrue(outcome.success)
self.assertEqual(client.calls.count(("search", "flaky")), 3)
self.assertEqual(delays, [0.5, 1.0])
async def test_transient_failure_exhausts_retries_and_reports_failure(self) -> None:
client = StubResearchClient()
client.failures_remaining[("search", "persistent")] = 100
frontier = QueryFrontier()
pipeline = ResearchPipeline(client, frontier)
with patch("typosaurus_sandbox.research.pipeline.asyncio.sleep", side_effect=_no_sleep):
outcome = await pipeline.process(WorkItem("web", "persistent"))
self.assertFalse(outcome.success)
self.assertEqual(outcome.status_code, 503)
self.assertEqual(outcome.error, "search failed")
self.assertEqual(client.calls.count(("search", "persistent")), RETRY_MAX_ATTEMPTS)
async def test_client_error_is_not_retried(self) -> None:
client = StubResearchClient()
client.error_on.add(("chat", "bad request"))
frontier = QueryFrontier()
pipeline = ResearchPipeline(client, frontier)
with patch("typosaurus_sandbox.research.pipeline.asyncio.sleep", side_effect=_no_sleep):
outcome = await pipeline.process(WorkItem("chat", "bad request"))
self.assertFalse(outcome.success)
self.assertEqual(outcome.status_code, 400)
self.assertEqual(client.calls.count(("chat", "bad request")), 1)
async def test_unexpected_exception_recorded_as_failure_outcome(self) -> None:
client = StubResearchClient()
client.explode_on.add(("search", "boom"))
frontier = QueryFrontier()
async def items() -> AsyncIterator[WorkItem]:
yield WorkItem("web", "boom")
yield WorkItem("web", "fine")
pipeline = ResearchPipeline(client, frontier)
report = await pipeline.run(items())
self.assertEqual(report.requests_succeeded, 1)
self.assertEqual(report.requests_failed, 1)
self.assertEqual(len(report.outcomes), 2)
failed = next(outcome for outcome in report.outcomes if not outcome.success)
self.assertEqual(failed.endpoint, "/search")
self.assertIsNone(failed.status_code)
self.assertIn("boom", failed.error or "")
self.assertEqual(failed.item.value, "boom")
if __name__ == "__main__":
unittest.main()