feat: add seo_meta service for AI-generated SEO metadata with CLI management and database layer

Implement a new `SeoMetaService` subservice that generates clean SEO title/description/keywords for published content items, distinct from the existing SEO diagnostics auditor. Add `seo_metadata` polymorphic table with soft-delete support, batch query methods, and usage tracking. Extend the CLI with `seo-meta prune` and `seo-meta clear` commands for job row lifecycle management. Wire `schedule_seo_meta_for_table` into content creation and editing flows in `content.py`. Document the new service in `AGENTS.md` and `README.md`, including the `extra_head` site setting for custom `<head>` injection.
2026-06-19 20:15:22 +00:00
parent 426d3639c6
commit d10f1af118
51 changed files with 2262 additions and 93 deletions
@@ -0,0 +1,84 @@
+# retoor <retoor@molodetz.nl>
+
+import json
+
+from tests.conftest import run_async
+
+from devplacepy.services.jobs.deepsearch import orchestrate as orchestrate_module
+from devplacepy.services.jobs.deepsearch.crawl import CrawledPage
+from devplacepy.services.jobs.deepsearch.orchestrate import Orchestration, orchestrate, source_diversity
+
+
+def _page(url, text="content " * 30):
+    return CrawledPage(url=url, title="T", text=text, source="httpx", status=200)
+
+
+def test_source_diversity_empty_is_zero():
+    assert source_diversity([]) == 0.0
+
+
+def test_source_diversity_increases_with_distinct_domains():
+    one = source_diversity([_page("https://a.example/1"), _page("https://a.example/2")])
+    many = source_diversity([_page("https://a.example/1"), _page("https://b.example/2")])
+    assert 0.0 < one <= 1.0
+    assert many > one
+
+
+def test_source_diversity_capped_at_one():
+    pages = [_page(f"https://d{i}.example") for i in range(8)]
+    assert source_diversity(pages) <= 1.0
+
+
+def test_orchestrate_with_no_pages_returns_gap():
+    async def fake_complete(messages, api_key, **kwargs):
+        return {}, {}, 0
+
+    result = run_async(orchestrate("q", [], "k", lambda frame: None))
+    assert isinstance(result, Orchestration)
+    assert result.source_diversity == 0.0
+    assert result.gaps
+
+
+def test_orchestrate_grounded_run_emits_agent_frames(monkeypatch):
+    frames = []
+
+    summary_payload = json.dumps(
+        {
+            "summary": "A grounded answer.",
+            "findings": [
+                {"title": "Finding", "detail": "Detail", "confidence": 0.7, "citations": [1]}
+            ],
+        }
+    )
+    gaps_payload = json.dumps({"gaps": ["one open question"]})
+    link_payload = json.dumps({"confidence": 0.8})
+    replies = iter([summary_payload, gaps_payload, link_payload])
+
+    async def fake_request_completion(messages, api_key, **kwargs):
+        return ({"choices": [{"message": {"content": next(replies)}}]}, {}, 5)
+
+    monkeypatch.setattr(orchestrate_module, "request_completion", fake_request_completion)
+
+    pages = [_page("https://a.example"), _page("https://b.example")]
+    result = run_async(orchestrate("question", pages, "k", frames.append))
+
+    assert result.summary == "A grounded answer."
+    assert result.findings and result.findings[0]["citations"] == [1]
+    assert result.gaps == ["one open question"]
+    assert 0.0 < result.confidence <= 1.0
+    assert result.source_diversity > 0.0
+    assert result.score > 0
+    agents = {f["agent"] for f in frames if f.get("type") == "agent"}
+    assert agents == {"summarizer", "critic", "linker"}
+
+
+def test_orchestrate_falls_back_to_heuristic_on_failure(monkeypatch):
+    async def boom(messages, api_key, **kwargs):
+        raise RuntimeError("upstream down")
+
+    monkeypatch.setattr(orchestrate_module, "request_completion", boom)
+    pages = [_page("https://a.example"), _page("https://b.example")]
+    result = run_async(orchestrate("question", pages, "k", lambda frame: None))
+    assert result.findings
+    assert result.gaps
+    assert result.source_diversity > 0.0
@@ -0,0 +1,42 @@
+# retoor <retoor@molodetz.nl>
+
+from devplacepy.services.jobs.deepsearch.phases import (
+    PHASE_ANALYSIS,
+    PHASE_CRAWLING,
+    PHASE_INDEXING,
+    PHASE_LABELS,
+    PHASE_ORDER,
+    PHASE_PLANNING,
+    PHASE_SEARCHING,
+    PHASE_SYNTHESIS,
+    TOTAL_PHASES,
+    phase_index,
+)
+
+
+def test_phase_order_is_full_pipeline_in_sequence():
+    assert PHASE_ORDER == [
+        PHASE_PLANNING,
+        PHASE_SEARCHING,
+        PHASE_CRAWLING,
+        PHASE_INDEXING,
+        PHASE_ANALYSIS,
+        PHASE_SYNTHESIS,
+    ]
+    assert TOTAL_PHASES == len(PHASE_ORDER)
+
+
+def test_phase_index_matches_position():
+    assert phase_index(PHASE_PLANNING) == 0
+    assert phase_index(PHASE_INDEXING) == 3
+    assert phase_index(PHASE_SYNTHESIS) == TOTAL_PHASES - 1
+
+
+def test_phase_index_unknown_falls_back_to_zero():
+    assert phase_index("not-a-phase") == 0
+    assert phase_index("") == 0
+
+
+def test_every_phase_has_a_label():
+    for phase in PHASE_ORDER:
+        assert PHASE_LABELS.get(phase)
@@ -92,6 +92,56 @@ def test_worker_run_produces_report(monkeypatch):
        VectorStore("ds_worker_test_one").drop()


+def test_index_chunks_emits_batch_progress(monkeypatch):
+    async def fake_embed_async(texts, api_key, **kwargs):
+        return local_embed(texts)
+
+    monkeypatch.setattr(worker_module, "embed_texts", fake_embed_async)
+    monkeypatch.setattr(worker_module, "EMBED_BATCH", 1)
+
+    frames = []
+    pages = [
+        CrawledPage(
+            url="https://example.com/a",
+            title="Page A",
+            text="The transistor was invented at Bell Labs. " * 30,
+            source="httpx",
+            status=200,
+        )
+    ]
+    store = VectorStore("ds_index_chunks_test")
+    try:
+        chunk_count, backend = run_async(
+            worker_module._index_chunks(store, pages, "k", frames.append)
+        )
+        assert chunk_count > 0
+        assert backend in ("gateway", "local")
+        batch_frames = [f for f in frames if f.get("type") == "embed_batch"]
+        assert batch_frames
+        assert all(f["total"] == chunk_count for f in batch_frames)
+        assert max(f["done"] for f in batch_frames) == chunk_count
+        done_frames = [f for f in frames if f.get("type") == "embed_done"]
+        assert done_frames and done_frames[-1]["chunk_count"] == chunk_count
+    finally:
+        store.drop()
+
+
+def test_index_chunks_empty_pages_emits_done(monkeypatch):
+    frames = []
+    store = VectorStore("ds_index_chunks_empty_test")
+    try:
+        chunk_count, backend = run_async(
+            worker_module._index_chunks(store, [], "k", frames.append)
+        )
+        assert chunk_count == 0
+        assert backend == "empty"
+        assert any(
+            f.get("type") == "embed_done" and f.get("chunk_count") == 0 for f in frames
+        )
+    finally:
+        store.drop()
+
+
 def test_worker_control_cancel_stops(monkeypatch):
    pages = [
        CrawledPage(url="https://x.example", title="X", text="content " * 40, source="httpx", status=200)