Add thread notifications, SEO topic pages, and fix quiz auto-advance

Notifications: a new "thread" type notifies every other commenter on a
post whenever anyone comments on it, disregarding reply hierarchy -
excluding the actor and whoever already got a comment/reply
notification for that same event, so no one is double-notified.
Implemented via a background-deferred fan-out mirroring the existing
mention-notification pattern.

SEO: discussion_forum_posting() now embeds up to 20 of a post's
comments as nested schema.org Comment entities (not just an aggregate
count), and a new /topics hub plus /topics/{topic} pages give the
feed's topic filter real, independently crawlable/indexable URLs -
/feed?topic=X was never indexable since its canonical strips the
query string back to bare /feed. Both are wired end to end (schemas,
Devii actions, docs API, sitemap, locustfile load-test coverage).

Quiz player: the auto-advance to the next question used to hide the
just-answered slide in the same tick as rendering the grade, so on
any multi-question quiz the Correct/Not correct feedback was never
actually visible before the view moved on. Delayed via setTimeout,
with the pending timer cleared on manual navigation and on
disconnect so it can't race or fire on a removed component.

Also includes other local changes already in progress in this
working tree before this session (messaging, push delivery,
deepsearch jobs, game economy, quiz builder) - verified by the full
suite passing (3467 tests) but not authored or individually reviewed
in this session.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01VL9Xn57W5UR3HZbbuuzxdK

tests/unit/services/jobs/deepsearch/enhance.py

@@ -0,0 +1,90 @@
+# retoor <retoor@molodetz.nl>
+
+import json
+
+from tests.conftest import run_async
+
+from devplacepy.services.jobs.deepsearch import enhance as enhance_module
+from devplacepy.services.jobs.deepsearch.enhance import plan_followup_queries
+
+
+class _FakeResponse:
+    def __init__(self, status_code, payload):
+        self.status_code = status_code
+        self._payload = payload
+
+    def json(self):
+        return self._payload
+
+
+class _FakeClient:
+    def __init__(self, response):
+        self._response = response
+
+    async def __aenter__(self):
+        return self
+
+    async def __aexit__(self, *exc):
+        return False
+
+    async def post(self, *args, **kwargs):
+        return self._response
+
+
+def _client_returning(response):
+    def factory(**kwargs):
+        return _FakeClient(response)
+
+    return factory
+
+
+def test_plan_followup_queries_empty_without_covered_titles():
+    result = run_async(plan_followup_queries("q", [], "k"))
+    assert result == []
+
+
+def test_plan_followup_queries_parses_gateway_response(monkeypatch):
+    payload = {"choices": [{"message": {"content": json.dumps({"queries": ["a", "b"]})}}]}
+    monkeypatch.setattr(
+        enhance_module.stealth,
+        "stealth_async_client",
+        _client_returning(_FakeResponse(200, payload)),
+    )
+    result = run_async(plan_followup_queries("q", ["Existing source"], "k"))
+    assert result == ["a", "b"]
+
+
+def test_plan_followup_queries_empty_when_sources_already_cover_question(monkeypatch):
+    payload = {"choices": [{"message": {"content": json.dumps({"queries": []})}}]}
+    monkeypatch.setattr(
+        enhance_module.stealth,
+        "stealth_async_client",
+        _client_returning(_FakeResponse(200, payload)),
+    )
+    result = run_async(plan_followup_queries("q", ["Existing source"], "k"))
+    assert result == []
+
+
+def test_plan_followup_queries_fails_soft_on_gateway_error(monkeypatch):
+    monkeypatch.setattr(
+        enhance_module.stealth,
+        "stealth_async_client",
+        _client_returning(_FakeResponse(500, {})),
+    )
+    result = run_async(plan_followup_queries("q", ["Existing source"], "k"))
+    assert result == []
+
+
+def test_plan_followup_queries_caps_at_max(monkeypatch):
+    payload = {
+        "choices": [
+            {"message": {"content": json.dumps({"queries": ["a", "b", "c", "d", "e"]})}}
+        ]
+    }
+    monkeypatch.setattr(
+        enhance_module.stealth,
+        "stealth_async_client",
+        _client_returning(_FakeResponse(200, payload)),
+    )
+    result = run_async(plan_followup_queries("q", ["Existing source"], "k"))
+    assert len(result) == enhance_module.MAX_FOLLOWUP_QUERIES

tests/unit/services/jobs/deepsearch/orchestrate.py

@@ -4,9 +4,16 @@ import json
 
 from tests.conftest import run_async
 
+from devplacepy.services.deepsearch.store import Chunk
 from devplacepy.services.jobs.deepsearch import orchestrate as orchestrate_module
 from devplacepy.services.jobs.deepsearch.crawl import CrawledPage
-from devplacepy.services.jobs.deepsearch.orchestrate import Orchestration, orchestrate, source_diversity
+from devplacepy.services.jobs.deepsearch.orchestrate import (
+    Orchestration,
+    _cosine,
+    _mmr_select,
+    orchestrate,
+    source_diversity,
+)
 
 
 def _page(url, text="content " * 30):
@@ -149,6 +156,67 @@ def test_linker_receives_full_source_list(monkeypatch):
         assert f"[{n}]" in seen["linker"]
 
 
+def test_cosine_identical_vectors_is_one():
+    assert round(_cosine([1.0, 0.0], [1.0, 0.0]), 6) == 1.0
+
+
+def test_cosine_orthogonal_vectors_is_zero():
+    assert _cosine([1.0, 0.0], [0.0, 1.0]) == 0.0
+
+
+def test_cosine_mismatched_or_empty_is_zero():
+    assert _cosine([], [1.0]) == 0.0
+    assert _cosine([1.0], [1.0, 0.0]) == 0.0
+
+
+def test_mmr_select_prefers_diverse_over_redundant():
+    query_vector = [1.0, 0.0, 0.0]
+    most_relevant = Chunk(uid="a", text="a", url="https://a", title="A", embedding=[0.9, 0.436, 0.0])
+    near_duplicate = Chunk(uid="b", text="b", url="https://a2", title="B", embedding=[0.85, 0.527, 0.0])
+    diverse = Chunk(uid="c", text="c", url="https://c", title="C", embedding=[0.85, 0.0, 0.527])
+    selected = _mmr_select([most_relevant, near_duplicate, diverse], query_vector, limit=2)
+    assert {chunk.uid for chunk in selected} == {"a", "c"}
+
+
+def test_mmr_select_falls_back_when_embeddings_missing():
+    chunks = [Chunk(uid="a", text="a", url="https://a", title="A")]
+    assert _mmr_select(chunks, [1.0, 0.0], limit=1) == chunks
+
+
+def test_mmr_select_empty_input():
+    assert _mmr_select([], [1.0, 0.0], limit=3) == []
+
+
+def test_orchestrate_grounded_run_includes_follow_up_questions(monkeypatch):
+    replies = iter(
+        [
+            "## Answer\nA grounded answer [1].",
+            json.dumps(
+                {
+                    "findings": [
+                        {"title": "Finding", "detail": "Detail", "confidence": 0.7, "citations": [1]}
+                    ]
+                }
+            ),
+            json.dumps({"confidence": 0.8}),
+            json.dumps({"questions": ["What about X?", "How does Y compare?"]}),
+        ]
+    )
+
+    async def fake_request_completion(messages, api_key, **kwargs):
+        return ({"choices": [{"message": {"content": next(replies)}}]}, {}, 5)
+
+    monkeypatch.setattr(orchestrate_module, "request_completion", fake_request_completion)
+    pages = [_page("https://a.example"), _page("https://b.example")]
+    result = run_async(orchestrate("question", pages, "k", lambda frame: None))
+    assert result.follow_up_questions == ["What about X?", "How does Y compare?"]
+
+
+def test_orchestrate_heuristic_path_has_no_follow_up_questions():
+    result = run_async(orchestrate("q", [], "k", lambda frame: None))
+    assert result.follow_up_questions == []
+
+
 def test_parse_json_tolerates_fences_and_trailing_garbage():
     from devplacepy.services.jobs.deepsearch.orchestrate import _parse_json
 

tests/unit/services/jobs/deepsearch/worker.py

@@ -20,13 +20,18 @@ def _patch_pipeline(monkeypatch, pages):
     async def fake_search(queries, emit=lambda frame: None):
         return [{"url": page.url, "title": page.title, "description": ""} for page in pages]
 
-    async def fake_crawl(candidates, max_pages, emit, is_cached, should_stop, query="", depth=1):
-        outcome = CrawlOutcome()
+    async def fake_crawl(
+        candidates, max_pages, emit, is_cached, should_stop, query="", depth=1, seen_hashes=None
+    ):
+        outcome = CrawlOutcome(seen_hashes=seen_hashes if seen_hashes is not None else set())
         for page in pages[:max_pages]:
             emit({"type": "page_loaded", "url": page.url, "done": 1, "total": len(pages)})
             outcome.pages.append(page)
         return outcome
 
+    async def fake_plan_followups(query, covered_titles, api_key, emit=lambda frame: None):
+        return []
+
     def fake_embed(texts, api_key):
         return local_embed(texts)
 
@@ -47,6 +52,7 @@ def _patch_pipeline(monkeypatch, pages):
     monkeypatch.setattr(worker_module, "plan_queries", fake_plan)
     monkeypatch.setattr(worker_module, "search_queries", fake_search)
     monkeypatch.setattr(worker_module, "crawl", fake_crawl)
+    monkeypatch.setattr(worker_module, "plan_followup_queries", fake_plan_followups)
     monkeypatch.setattr(worker_module, "embed_texts", fake_embed_async)
     monkeypatch.setattr(worker_module, "orchestrate", fake_orchestrate)
 
@@ -141,6 +147,78 @@ def test_index_chunks_empty_pages_emits_done(monkeypatch):
         store.drop()
 
 
+def test_worker_run_performs_refinement_round_when_budget_remains(monkeypatch):
+    initial_page = CrawledPage(
+        url="https://example.com/a",
+        title="Page A",
+        text="The transistor was invented at Bell Labs. " * 20,
+        source="httpx",
+        status=200,
+    )
+    refined_page = CrawledPage(
+        url="https://other.example/b",
+        title="Page B",
+        text="Semiconductors are made from silicon. " * 20,
+        source="httpx",
+        status=200,
+    )
+    _patch_pipeline(monkeypatch, [initial_page])
+
+    followup_calls = []
+
+    async def fake_plan_followups_once(query, covered_titles, api_key, emit=lambda frame: None):
+        if followup_calls:
+            return []
+        followup_calls.append(covered_titles)
+        return ["a more specific angle"]
+
+    async def fake_search_followup(queries, emit=lambda frame: None):
+        return [{"url": refined_page.url, "title": refined_page.title, "description": ""}]
+
+    async def fake_crawl_refinement(
+        candidates, max_pages, emit, is_cached, should_stop, query="", depth=1, seen_hashes=None
+    ):
+        outcome = CrawlOutcome(seen_hashes=seen_hashes if seen_hashes is not None else set())
+        outcome.pages.append(refined_page)
+        return outcome
+
+    monkeypatch.setattr(worker_module, "plan_followup_queries", fake_plan_followups_once)
+
+    real_search_queries = worker_module.search_queries
+    real_crawl = worker_module.crawl
+    call_count = {"search": 0, "crawl": 0}
+
+    async def routed_search(queries, emit=lambda frame: None):
+        call_count["search"] += 1
+        if call_count["search"] == 1:
+            return await real_search_queries(queries, emit)
+        return await fake_search_followup(queries, emit)
+
+    async def routed_crawl(*args, **kwargs):
+        call_count["crawl"] += 1
+        if call_count["crawl"] == 1:
+            return await real_crawl(*args, **kwargs)
+        return await fake_crawl_refinement(*args, **kwargs)
+
+    monkeypatch.setattr(worker_module, "search_queries", routed_search)
+    monkeypatch.setattr(worker_module, "crawl", routed_crawl)
+
+    with tempfile.TemporaryDirectory() as tmp:
+        output_dir = Path(tmp)
+        payload = {
+            "query": "history of the transistor",
+            "max_pages": 5,
+            "depth": 2,
+            "api_key": "k",
+            "collection": "ds_worker_refine_test",
+            "cached_hashes": [],
+        }
+        report = run_async(worker_module._run(payload, output_dir))
+        assert report["page_count"] == 2
+        assert followup_calls and followup_calls[0] == ["Page A"]
+        VectorStore("ds_worker_refine_test").drop()
+
+
 def test_worker_control_cancel_stops(monkeypatch):
     pages = [
         CrawledPage(url="https://x.example", title="X", text="content " * 40, source="httpx", status=200)