Update

tests/unit/services/jobs/deepsearch/crawl.py

@@ -0,0 +1,112 @@
+# retoor <retoor@molodetz.nl>
+
+from tests.conftest import run_async
+
+from devplacepy.services.jobs.deepsearch import crawl as crawl_module
+from devplacepy.services.jobs.deepsearch.crawl import (
+    CrawledPage,
+    _interleave,
+    _is_hostile,
+    _snippet_page,
+    crawl,
+)
+
+
+async def _no_stop() -> bool:
+    return False
+
+
+def test_interleave_round_robins_and_dedupes():
+    buckets = [
+        [{"url": "a"}, {"url": "b"}],
+        [{"url": "c"}, {"url": "a"}],
+        [{"url": "d"}],
+    ]
+    assert [item["url"] for item in _interleave(buckets)] == ["a", "c", "d", "b"]
+
+
+def test_is_hostile_matches_social_domains():
+    assert _is_hostile("https://x.com/user/status/1")
+    assert _is_hostile("https://www.youtube.com/watch?v=abc")
+    assert _is_hostile("https://old.reddit.com/r/x")
+    assert not _is_hostile("https://blog.example.com/post")
+
+
+def test_snippet_page_prefers_content_over_description():
+    candidate = {
+        "url": "https://x.com/a/status/1",
+        "title": "Tweet",
+        "description": "short",
+        "content": "This is the full rsearch content, deliberately written long enough to clear the snippet minimum length floor so that it is kept as a real source easily.",
+    }
+    page = _snippet_page(candidate, 0)
+    assert page is not None
+    assert page.source == "search"
+    assert "full rsearch content" in page.text
+
+
+def test_snippet_page_none_when_too_thin():
+    assert _snippet_page({"url": "https://x.com/a", "content": "tiny"}, 0) is None
+
+
+def test_crawl_uses_snippet_for_hostile_and_skips_fetch(monkeypatch):
+    fetched = []
+
+    async def fake_fetch(url, depth):
+        fetched.append(url)
+        return CrawledPage(url=url, title="T", text="x " * 200, source="httpx", status=200, depth=depth)
+
+    monkeypatch.setattr(crawl_module, "fetch_page", fake_fetch)
+    candidates = [
+        {
+            "url": "https://x.com/ThePrimeagen/status/1",
+            "title": "Prime",
+            "description": "",
+            "content": "The real tweet text returned by rsearch for this social post, well past the snippet minimum length threshold so it survives as a usable source here.",
+        }
+    ]
+    outcome = run_async(
+        crawl(candidates, 6, lambda f: None, lambda u: False, _no_stop, query="q", depth=1)
+    )
+    assert fetched == []
+    assert len(outcome.pages) == 1
+    assert outcome.pages[0].source == "search"
+
+
+def test_crawl_prefers_richer_crawl_over_snippet(monkeypatch):
+    async def fake_fetch(url, depth):
+        return CrawledPage(url=url, title="Article", text="Deep article body. " * 60, source="httpx", status=200, depth=depth)
+
+    monkeypatch.setattr(crawl_module, "fetch_page", fake_fetch)
+    candidates = [
+        {
+            "url": "https://blog.example.com/a",
+            "title": "Blog",
+            "description": "",
+            "content": "A short snippet that is longer than the floor but shorter than the crawled article body itself.",
+        }
+    ]
+    outcome = run_async(
+        crawl(candidates, 6, lambda f: None, lambda u: False, _no_stop, query="q", depth=1)
+    )
+    assert outcome.pages[0].source == "httpx"
+
+
+def test_crawl_falls_back_to_snippet_when_fetch_empty(monkeypatch):
+    async def fake_fetch(url, depth):
+        return None
+
+    monkeypatch.setattr(crawl_module, "fetch_page", fake_fetch)
+    candidates = [
+        {
+            "url": "https://walled.example.com/a",
+            "title": "Wall",
+            "description": "",
+            "content": "The search snippet holds the real content that the login-walled page refused to serve to the bot today, and it is comfortably past the snippet minimum length floor.",
+        }
+    ]
+    outcome = run_async(
+        crawl(candidates, 6, lambda f: None, lambda u: False, _no_stop, query="q", depth=1)
+    )
+    assert len(outcome.pages) == 1
+    assert outcome.pages[0].source == "search"

tests/unit/services/jobs/deepsearch/extract.py

@@ -0,0 +1,67 @@
+# retoor <retoor@molodetz.nl>
+
+from devplacepy.services.jobs.deepsearch.extract import extract_html, relevant_links
+
+PAGE = """
+<html><head><title>Framework Trends - Blog</title></head><body>
+<header><a href="/">Home</a> <a href="/about">About</a> <a href="/login">Login</a></header>
+<nav><ul><li><a href="/blog">Blog</a></li><li><a href="/dev">Dev</a></li></ul></nav>
+<div role="banner">We use cookies to improve your experience on this website today.</div>
+<main><article>
+<h1>Trends that define web development</h1>
+<p>Server components became the default rendering model, cutting client bundles by forty percent according to the survey.</p>
+<p>Signals-based reactivity landed in the standards pipeline; see the <a href="/signals-deep-dive">signals deep dive</a> article.</p>
+</article></main>
+<footer><p>Copyright. All rights reserved. Privacy. Terms. Subscribe to our newsletter now.</p></footer>
+</body></html>
+"""
+
+
+def test_extract_prefers_article_content_and_drops_chrome():
+    page = extract_html(PAGE, base_url="https://blog.example.com/trends/")
+    assert page.title == "Framework Trends - Blog"
+    assert "Server components" in page.text
+    assert "Signals-based reactivity" in page.text
+    assert "cookies" not in page.text
+    assert "Copyright" not in page.text
+    assert "Home" not in page.text
+
+
+def test_extract_emits_blank_line_paragraphs():
+    page = extract_html(PAGE, base_url="https://blog.example.com/trends/")
+    assert "\n\n" in page.text
+
+
+def test_extract_resolves_links_absolute_and_skips_nav():
+    page = extract_html(PAGE, base_url="https://blog.example.com/trends/")
+    urls = [url for url, _text in page.links]
+    assert "https://blog.example.com/signals-deep-dive" in urls
+    assert "https://blog.example.com/blog" not in urls
+
+
+def test_extract_unescapes_entities():
+    page = extract_html(
+        "<html><body><p>Ampersand &amp; arrow &#8594; and more text to pass the length gate.</p></body></html>"
+    )
+    assert "&amp;" not in page.text
+    assert "&#8594;" not in page.text
+    assert "&" in page.text
+
+
+def test_extract_survives_malformed_html():
+    page = extract_html("<div><p>Unclosed paragraph with enough characters to be kept around.<div></span>")
+    assert "Unclosed paragraph" in page.text
+
+
+def test_relevant_links_scores_by_query_overlap():
+    links = [
+        ("https://a.example/web-frameworks-2026", "web frameworks in 2026"),
+        ("https://a.example/cookie-policy", "cookie policy"),
+        ("https://a.example/logo.png", "frameworks logo"),
+    ]
+    picked = relevant_links(links, "latest web frameworks 2026", 2)
+    assert picked == ["https://a.example/web-frameworks-2026"]
+
+
+def test_relevant_links_empty_query_returns_nothing():
+    assert relevant_links([("https://a.example/x", "text")], "", 3) == []

tests/unit/services/jobs/deepsearch/orchestrate.py

@@ -29,30 +29,27 @@ def test_source_diversity_capped_at_one():
     assert source_diversity(pages) <= 1.0
 
 
-def test_orchestrate_with_no_pages_returns_gap():
-    async def fake_complete(messages, api_key, **kwargs):
-        return {}, {}, 0
-
+def test_orchestrate_with_no_pages_is_heuristic():
     result = run_async(orchestrate("q", [], "k", lambda frame: None))
     assert isinstance(result, Orchestration)
     assert result.source_diversity == 0.0
-    assert result.gaps
+    assert result.synthesis == "heuristic"
+    assert not result.findings
 
 
 def test_orchestrate_grounded_run_emits_agent_frames(monkeypatch):
     frames = []
 
-    summary_payload = json.dumps(
+    report_payload = "## Answer\nA grounded answer [1]."
+    findings_payload = json.dumps(
         {
-            "summary": "A grounded answer.",
             "findings": [
                 {"title": "Finding", "detail": "Detail", "confidence": 0.7, "citations": [1]}
-            ],
+            ]
         }
     )
-    gaps_payload = json.dumps({"gaps": ["one open question"]})
     link_payload = json.dumps({"confidence": 0.8})
-    replies = iter([summary_payload, gaps_payload, link_payload])
+    replies = iter([report_payload, findings_payload, link_payload])
 
     async def fake_request_completion(messages, api_key, **kwargs):
         return ({"choices": [{"message": {"content": next(replies)}}]}, {}, 5)
@@ -62,14 +59,14 @@ def test_orchestrate_grounded_run_emits_agent_frames(monkeypatch):
     pages = [_page("https://a.example"), _page("https://b.example")]
     result = run_async(orchestrate("question", pages, "k", frames.append))
 
-    assert result.summary == "A grounded answer."
+    assert result.summary == "## Answer\nA grounded answer [1]."
     assert result.findings and result.findings[0]["citations"] == [1]
-    assert result.gaps == ["one open question"]
+    assert result.synthesis == "agents"
     assert 0.0 < result.confidence <= 1.0
     assert result.source_diversity > 0.0
     assert result.score > 0
     agents = {f["agent"] for f in frames if f.get("type") == "agent"}
-    assert agents == {"summarizer", "critic", "linker"}
+    assert agents == {"summarizer", "extractor", "linker"}
 
 
 def test_orchestrate_falls_back_to_heuristic_on_failure(monkeypatch):
@@ -77,8 +74,85 @@ def test_orchestrate_falls_back_to_heuristic_on_failure(monkeypatch):
         raise RuntimeError("upstream down")
 
     monkeypatch.setattr(orchestrate_module, "request_completion", boom)
+    frames = []
+    pages = [_page("https://a.example"), _page("https://b.example")]
+    result = run_async(orchestrate("question", pages, "k", frames.append))
+    assert result.findings
+    assert result.synthesis == "heuristic"
+    assert any(f.get("status") == "failed" for f in frames if f.get("type") == "agent")
+    assert result.source_diversity > 0.0
+
+
+def test_orchestrate_survives_linker_failure(monkeypatch):
+    replies = iter(
+        [
+            "A full report [1].",
+            json.dumps(
+                {
+                    "findings": [
+                        {"title": "F", "detail": "D", "confidence": 0.5, "citations": [1]}
+                    ]
+                }
+            ),
+        ]
+    )
+
+    async def flaky(messages, api_key, **kwargs):
+        try:
+            content = next(replies)
+        except StopIteration:
+            raise RuntimeError("upstream down")
+        return ({"choices": [{"message": {"content": content}}]}, {}, 5)
+
+    monkeypatch.setattr(orchestrate_module, "request_completion", flaky)
     pages = [_page("https://a.example"), _page("https://b.example")]
     result = run_async(orchestrate("question", pages, "k", lambda frame: None))
+    assert result.synthesis == "agents"
+    assert result.summary == "A full report [1]."
     assert result.findings
-    assert result.gaps
-    assert result.source_diversity > 0.0
+    assert result.confidence >= 0.35
+
+
+def test_numbered_source_digest_keeps_every_source_number_under_cap():
+    from devplacepy.services.jobs.deepsearch.orchestrate import _numbered_source_digest
+    import re
+
+    pages = [_page(f"https://s{i}.example", text="word " * 500) for i in range(1, 13)]
+    digest = _numbered_source_digest(pages, per_source=600, cap=4000)
+    present = {int(n) for n in re.findall(r"\[(\d+)\]", digest)}
+    assert present == set(range(1, 13))
+
+
+def test_linker_receives_full_source_list(monkeypatch):
+    seen = {}
+    replies = iter(
+        [
+            "Report body [1].",
+            json.dumps(
+                {"findings": [{"title": "F", "detail": "D", "confidence": 1.0, "citations": [2]}]}
+            ),
+            json.dumps({"confidence": 0.9}),
+        ]
+    )
+
+    async def capture(messages, api_key, **kwargs):
+        content = messages[-1]["content"]
+        if content.startswith("FINDINGS:") and "SOURCES:" in content:
+            seen["linker"] = content
+        return ({"choices": [{"message": {"content": next(replies)}}]}, {}, 5)
+
+    monkeypatch.setattr(orchestrate_module, "request_completion", capture)
+    pages = [_page(f"https://s{i}.example") for i in range(1, 13)]
+    result = run_async(orchestrate("q", pages, "k", lambda f: None))
+    assert result.confidence >= 0.35
+    for n in range(1, 13):
+        assert f"[{n}]" in seen["linker"]
+
+
+def test_parse_json_tolerates_fences_and_trailing_garbage():
+    from devplacepy.services.jobs.deepsearch.orchestrate import _parse_json
+
+    assert _parse_json('```json\n{"gaps": ["g"]}\n```') == {"gaps": ["g"]}
+    assert _parse_json('prose {"confidence": 0.7} more prose') == {"confidence": 0.7}
+    assert _parse_json('{"gaps": ["a"]} {"broken": ') == {"gaps": ["a"]}
+    assert _parse_json("no json here") == {}

tests/unit/services/jobs/deepsearch/worker.py

@@ -20,7 +20,7 @@ def _patch_pipeline(monkeypatch, pages):
     async def fake_search(queries, emit=lambda frame: None):
         return [{"url": page.url, "title": page.title, "description": ""} for page in pages]
 
-    async def fake_crawl(candidates, max_pages, emit, is_cached, should_stop):
+    async def fake_crawl(candidates, max_pages, emit, is_cached, should_stop, query="", depth=1):
         outcome = CrawlOutcome()
         for page in pages[:max_pages]:
             emit({"type": "page_loaded", "url": page.url, "done": 1, "total": len(pages)})
@@ -33,13 +33,12 @@ def _patch_pipeline(monkeypatch, pages):
     async def fake_embed_async(texts, api_key, **kwargs):
         return local_embed(texts)
 
-    async def fake_orchestrate(question, crawled, api_key, emit):
+    async def fake_orchestrate(question, crawled, api_key, emit, store=None, queries=None):
         from devplacepy.services.jobs.deepsearch.orchestrate import Orchestration
 
         return Orchestration(
             summary="A summary.",
             findings=[{"title": "F", "detail": "D", "confidence": 0.5, "citations": [1]}],
-            gaps=["gap"],
             confidence=0.6,
             source_diversity=0.5,
             score=42,