feat: add seo_meta service for AI-generated SEO metadata with CLI management and database layer

Implement a new `SeoMetaService` subservice that generates clean SEO title/description/keywords for published content items, distinct from the existing SEO diagnostics auditor. Add `seo_metadata` polymorphic table with soft-delete support, batch query methods, and usage tracking. Extend the CLI with `seo-meta prune` and `seo-meta clear` commands for job row lifecycle management. Wire `schedule_seo_meta_for_table` into content creation and editing flows in `content.py`. Document the new service in `AGENTS.md` and `README.md`, including the `extra_head` site setting for custom `<head>` injection.
2026-06-19 20:15:22 +00:00
parent 426d3639c6
commit d10f1af118
51 changed files with 2262 additions and 93 deletions
@@ -259,6 +259,116 @@ def test_add_issue_usage_ignores_zero_call_batches(local_db):
    assert get_issue_usage()["calls"] == 0


+def test_seo_usage_accumulates_and_computes_averages(local_db):
+    from devplacepy.database import add_seo_usage, get_seo_usage
+
+    get_table("seo_usage").delete()
+    empty = get_seo_usage()
+    assert empty["calls"] == 0
+    assert empty["avg_tokens"] == 0.0
+    assert empty["avg_cost_usd"] == 0.0
+
+    totals = {
+        "calls": 2,
+        "prompt_tokens": 2000,
+        "completion_tokens": 400,
+        "total_tokens": 2400,
+        "cost_usd": 0.0005,
+        "upstream_latency_ms": 1600.0,
+        "total_latency_ms": 1800.0,
+    }
+    add_seo_usage(totals)
+    add_seo_usage(totals)
+
+    usage = get_seo_usage()
+    assert usage["calls"] == 4
+    assert usage["total_tokens"] == 4800
+    assert abs(usage["cost_usd"] - 0.001) < 1e-9
+    assert usage["avg_tokens"] == 1200.0
+    assert usage["avg_cost_usd"] == 0.00025
+    assert usage["avg_tokens_per_second"] == 250.0
+
+
+def test_add_seo_usage_ignores_zero_call_batches(local_db):
+    from devplacepy.database import add_seo_usage, get_seo_usage
+
+    get_table("seo_usage").delete()
+    add_seo_usage({"calls": 0, "total_tokens": 999, "cost_usd": 1.0})
+    assert get_seo_usage()["calls"] == 0
+
+
+def test_upsert_seo_metadata_inserts_then_updates_single_row(local_db):
+    from devplacepy.database import (
+        get_seo_metadata,
+        has_fresh_seo_metadata,
+        upsert_seo_metadata,
+    )
+
+    target_uid = generate_uid()
+    table = get_table("seo_metadata")
+
+    upsert_seo_metadata(
+        "post", target_uid, "Plain Title", "Plain description", "alpha, beta", "pending", "plain"
+    )
+    assert has_fresh_seo_metadata("post", target_uid) is False
+    assert get_seo_metadata("post", target_uid) is None
+
+    upsert_seo_metadata(
+        "post", target_uid, "AI Title", "AI description", "python, sqlite", "ready", "ai"
+    )
+    ready = get_seo_metadata("post", target_uid)
+    assert ready is not None
+    assert ready["seo_title"] == "AI Title"
+    assert ready["seo_keywords"] == "python, sqlite"
+    assert ready["generated_at"]
+    assert ready["deleted_at"] is None
+    assert has_fresh_seo_metadata("post", target_uid) is True
+
+    rows = list(table.find(target_type="post", target_uid=target_uid))
+    assert len(rows) == 1
+
+
+def test_seo_metadata_batch_returns_only_ready_live_rows(local_db):
+    from devplacepy.database import (
+        get_seo_metadata_batch,
+        upsert_seo_metadata,
+    )
+
+    ready_uid = generate_uid()
+    pending_uid = generate_uid()
+    upsert_seo_metadata("gist", ready_uid, "T", "D", "k1, k2", "ready", "ai")
+    upsert_seo_metadata("gist", pending_uid, "T", "D", "k1, k2", "pending", "plain")
+
+    batch = get_seo_metadata_batch("gist", [ready_uid, pending_uid])
+    assert ready_uid in batch
+    assert pending_uid not in batch
+    assert batch[ready_uid]["status"] == "ready"
+
+
+def test_mark_seo_metadata_stale_drops_ready_status(local_db):
+    from devplacepy.database import (
+        get_seo_metadata,
+        mark_seo_metadata_stale,
+        upsert_seo_metadata,
+    )
+
+    target_uid = generate_uid()
+    upsert_seo_metadata("project", target_uid, "T", "D", "k", "ready", "ai")
+    assert get_seo_metadata("project", target_uid) is not None
+
+    mark_seo_metadata_stale("project", target_uid)
+    assert get_seo_metadata("project", target_uid) is None
+
+
+def test_seo_metadata_is_soft_deletable_and_indexed(local_db):
+    from devplacepy.database import SOFT_DELETE_TABLES
+
+    assert "seo_metadata" in SOFT_DELETE_TABLES
+    columns = get_table("seo_metadata").columns
+    assert "deleted_at" in columns
+    assert "deleted_by" in columns
+
+
 def test_interleave_by_author_spreads_consecutive_runs():
    from devplacepy.database import interleave_by_author

@@ -5,6 +5,14 @@ import devplacepy.seo as seo
 import devplacepy.attachments as attach


+def _seo_request():
+    return types.SimpleNamespace(
+        base_url="https://x.test/",
+        url=types.SimpleNamespace(path="/posts/abc"),
+        query_params={},
+    )
+
+
 def test_truncate_no_overflow():
    assert len(seo.truncate("x" * 200)) <= 160
    assert seo.truncate("short text") == "short text"
@@ -76,3 +84,69 @@ def test_detect_mime_neutralizes_dangerous_types():
    assert attach._detect_mime(b"", "x.svg") == "application/octet-stream"
    assert attach._detect_mime(b"", "x.html") == "application/octet-stream"
    assert attach._detect_mime(b"", "x.png") == "image/png"
+
+
+def test_base_seo_context_emits_keywords_key():
+    ctx = seo.base_seo_context(
+        _seo_request(),
+        title="A Post",
+        description="Body text",
+        keywords="python, sqlite",
+    )
+    assert "meta_keywords" in ctx
+    assert ctx["meta_keywords"] == "python, sqlite"
+
+
+def test_base_seo_context_default_keywords_empty_without_target():
+    ctx = seo.base_seo_context(
+        _seo_request(), title="A Post", description="Body text"
+    )
+    assert ctx["meta_keywords"] == ""
+
+
+def test_base_seo_context_description_strips_markdown(monkeypatch):
+    monkeypatch.setattr(seo, "_ready_seo_metadata", lambda target: None)
+    ctx = seo.base_seo_context(
+        _seo_request(),
+        title="A Post",
+        description="## Heading\n\nThis is **bold** body with `code`.",
+    )
+    assert "##" not in ctx["meta_description"]
+    assert "**" not in ctx["meta_description"]
+    assert "`" not in ctx["meta_description"]
+    assert "bold" in ctx["meta_description"]
+
+
+def test_base_seo_context_plain_default_fills_fields_for_target(monkeypatch):
+    monkeypatch.setattr(seo, "_ready_seo_metadata", lambda target: None)
+    ctx = seo.base_seo_context(
+        _seo_request(),
+        title="Async Python Database Tooling",
+        description="A **guide** to async python database tooling with sqlite.",
+        seo_target=("post", "abc"),
+    )
+    assert ctx["meta_keywords"]
+    assert "python" in ctx["meta_keywords"]
+    assert ctx["meta_description"]
+    assert "**" not in ctx["meta_description"]
+
+
+def test_base_seo_context_consumes_ready_metadata(monkeypatch):
+    monkeypatch.setattr(
+        seo,
+        "_ready_seo_metadata",
+        lambda target: {
+            "seo_title": "Generated Title",
+            "seo_description": "Generated clean description",
+            "seo_keywords": "generated, metadata",
+        },
+    )
+    ctx = seo.base_seo_context(
+        _seo_request(),
+        title="Raw **markdown** title",
+        description="raw markdown body",
+        seo_target=("post", "abc"),
+    )
+    assert "Generated Title" in ctx["page_title"]
+    assert ctx["meta_description"] == "Generated clean description"
+    assert ctx["meta_keywords"] == "generated, metadata"
@@ -0,0 +1,101 @@
+# retoor <retoor@molodetz.nl>
+
+from devplacepy.seo_meta_text import (
+    DESCRIPTION_MAX,
+    KEYWORDS_MAX,
+    TITLE_MAX,
+    clamp_generated,
+    derive_keywords,
+    plain_seo_defaults,
+    plain_text_from_markdown,
+)
+
+
+def test_plain_text_strips_markdown_markup():
+    out = plain_text_from_markdown("# Heading\n\n**bold** and `code` and [link](http://x.test)")
+    assert "#" not in out
+    assert "**" not in out
+    assert "`" not in out
+    assert "](" not in out
+    assert "bold" in out
+    assert "link" in out
+
+
+def test_plain_seo_defaults_always_filled():
+    result = plain_seo_defaults("", "")
+    assert result["title"]
+    assert result["description"]
+    assert set(result) == {"title", "description", "keywords"}
+
+
+def test_plain_seo_defaults_uses_title_when_body_empty():
+    result = plain_seo_defaults("My Great Title", "")
+    assert result["title"] == "My Great Title"
+    assert result["description"] == "My Great Title"
+
+
+def test_plain_seo_defaults_description_has_no_markdown():
+    body = "## Section\n\nThis is **important** content with a [ref](http://x.test) inside."
+    result = plain_seo_defaults("Title here", body)
+    assert "##" not in result["description"]
+    assert "**" not in result["description"]
+    assert "](" not in result["description"]
+    assert "important" in result["description"]
+
+
+def test_plain_seo_defaults_clamps_lengths():
+    long_title = "word " * 40
+    long_body = "sentence body text " * 60
+    result = plain_seo_defaults(long_title, long_body)
+    assert len(result["title"]) <= TITLE_MAX
+    assert len(result["description"]) <= DESCRIPTION_MAX
+
+
+def test_plain_seo_defaults_keywords_filled_from_content():
+    result = plain_seo_defaults(
+        "Async Python Database Tooling",
+        "A guide to building async python database tooling with sqlite.",
+    )
+    assert result["keywords"]
+    terms = [term for term in result["keywords"].split(", ") if term]
+    assert "python" in terms
+    assert "the" not in terms
+    assert "a" not in terms
+
+
+def test_derive_keywords_dedupes_and_skips_stopwords():
+    keywords = derive_keywords("Python python", "the python and the database database engine")
+    terms = keywords.split(", ")
+    assert terms.count("python") == 1
+    assert "the" not in terms
+    assert "and" not in terms
+    assert "database" in terms
+
+
+def test_derive_keywords_respects_limit():
+    body = " ".join(f"keyword{n}" for n in range(40))
+    keywords = derive_keywords("title", body)
+    terms = [term for term in keywords.split(", ") if term]
+    assert len(terms) <= KEYWORDS_MAX
+
+
+def test_clamp_generated_strips_markup_and_normalizes_keywords():
+    result = clamp_generated(
+        "**Clean** Title",
+        "Some `markdown` description here.",
+        "Python, Python,  SQLite ;async\nweb",
+    )
+    assert "**" not in result["title"]
+    assert "`" not in result["description"]
+    terms = result["keywords"].split(", ")
+    assert terms.count("python") == 1
+    assert "sqlite" in terms
+    assert "async" in terms
+    assert "web" in terms
+
+
+def test_clamp_generated_honors_keyword_cap():
+    raw = ", ".join(f"term{n}" for n in range(40))
+    result = clamp_generated("t", "d", raw)
+    terms = [term for term in result["keywords"].split(", ") if term]
+    assert len(terms) <= KEYWORDS_MAX
@@ -50,3 +50,44 @@ def test_keyword_scores_rank_match_higher():
    assert scores
    best = max(scores, key=scores.get)
    assert "silicon" in next(c.text for c in chunks if c.uid == best).lower()
+
+
+def test_dims_reflects_embedding_width():
+    store = VectorStore("ds_store_test_dims")
+    try:
+        chunks = _chunks()
+        vectors = local_embed([c.text for c in chunks]).vectors
+        store.add(chunks, vectors)
+        assert store.dims == len(vectors[0])
+    finally:
+        store.drop()
+
+
+def test_coverage_analytics_empty_collection():
+    store = VectorStore("ds_store_test_cov_empty")
+    try:
+        analytics = store.coverage_analytics()
+        assert analytics == {
+            "chunks": 0,
+            "domains": 0,
+            "sources": 0,
+            "avg_chunk_chars": 0,
+        }
+    finally:
+        store.drop()
+
+
+def test_coverage_analytics_counts_chunks_and_sources():
+    store = VectorStore("ds_store_test_cov")
+    try:
+        chunks = _chunks()
+        for index, chunk in enumerate(chunks):
+            chunk.source = "httpx" if index == 0 else "playwright"
+        vectors = local_embed([c.text for c in chunks]).vectors
+        store.add(chunks, vectors)
+        analytics = store.coverage_analytics()
+        assert analytics["chunks"] == 3
+        assert analytics["sources"] == 2
+        assert analytics["avg_chunk_chars"] > 0
+    finally:
+        store.drop()
@@ -0,0 +1,84 @@
+# retoor <retoor@molodetz.nl>
+
+import json
+
+from tests.conftest import run_async
+
+from devplacepy.services.jobs.deepsearch import orchestrate as orchestrate_module
+from devplacepy.services.jobs.deepsearch.crawl import CrawledPage
+from devplacepy.services.jobs.deepsearch.orchestrate import Orchestration, orchestrate, source_diversity
+
+
+def _page(url, text="content " * 30):
+    return CrawledPage(url=url, title="T", text=text, source="httpx", status=200)
+
+
+def test_source_diversity_empty_is_zero():
+    assert source_diversity([]) == 0.0
+
+
+def test_source_diversity_increases_with_distinct_domains():
+    one = source_diversity([_page("https://a.example/1"), _page("https://a.example/2")])
+    many = source_diversity([_page("https://a.example/1"), _page("https://b.example/2")])
+    assert 0.0 < one <= 1.0
+    assert many > one
+
+
+def test_source_diversity_capped_at_one():
+    pages = [_page(f"https://d{i}.example") for i in range(8)]
+    assert source_diversity(pages) <= 1.0
+
+
+def test_orchestrate_with_no_pages_returns_gap():
+    async def fake_complete(messages, api_key, **kwargs):
+        return {}, {}, 0
+
+    result = run_async(orchestrate("q", [], "k", lambda frame: None))
+    assert isinstance(result, Orchestration)
+    assert result.source_diversity == 0.0
+    assert result.gaps
+
+
+def test_orchestrate_grounded_run_emits_agent_frames(monkeypatch):
+    frames = []
+
+    summary_payload = json.dumps(
+        {
+            "summary": "A grounded answer.",
+            "findings": [
+                {"title": "Finding", "detail": "Detail", "confidence": 0.7, "citations": [1]}
+            ],
+        }
+    )
+    gaps_payload = json.dumps({"gaps": ["one open question"]})
+    link_payload = json.dumps({"confidence": 0.8})
+    replies = iter([summary_payload, gaps_payload, link_payload])
+
+    async def fake_request_completion(messages, api_key, **kwargs):
+        return ({"choices": [{"message": {"content": next(replies)}}]}, {}, 5)
+
+    monkeypatch.setattr(orchestrate_module, "request_completion", fake_request_completion)
+
+    pages = [_page("https://a.example"), _page("https://b.example")]
+    result = run_async(orchestrate("question", pages, "k", frames.append))
+
+    assert result.summary == "A grounded answer."
+    assert result.findings and result.findings[0]["citations"] == [1]
+    assert result.gaps == ["one open question"]
+    assert 0.0 < result.confidence <= 1.0
+    assert result.source_diversity > 0.0
+    assert result.score > 0
+    agents = {f["agent"] for f in frames if f.get("type") == "agent"}
+    assert agents == {"summarizer", "critic", "linker"}
+
+
+def test_orchestrate_falls_back_to_heuristic_on_failure(monkeypatch):
+    async def boom(messages, api_key, **kwargs):
+        raise RuntimeError("upstream down")
+
+    monkeypatch.setattr(orchestrate_module, "request_completion", boom)
+    pages = [_page("https://a.example"), _page("https://b.example")]
+    result = run_async(orchestrate("question", pages, "k", lambda frame: None))
+    assert result.findings
+    assert result.gaps
+    assert result.source_diversity > 0.0
@@ -0,0 +1,42 @@
+# retoor <retoor@molodetz.nl>
+
+from devplacepy.services.jobs.deepsearch.phases import (
+    PHASE_ANALYSIS,
+    PHASE_CRAWLING,
+    PHASE_INDEXING,
+    PHASE_LABELS,
+    PHASE_ORDER,
+    PHASE_PLANNING,
+    PHASE_SEARCHING,
+    PHASE_SYNTHESIS,
+    TOTAL_PHASES,
+    phase_index,
+)
+
+
+def test_phase_order_is_full_pipeline_in_sequence():
+    assert PHASE_ORDER == [
+        PHASE_PLANNING,
+        PHASE_SEARCHING,
+        PHASE_CRAWLING,
+        PHASE_INDEXING,
+        PHASE_ANALYSIS,
+        PHASE_SYNTHESIS,
+    ]
+    assert TOTAL_PHASES == len(PHASE_ORDER)
+
+
+def test_phase_index_matches_position():
+    assert phase_index(PHASE_PLANNING) == 0
+    assert phase_index(PHASE_INDEXING) == 3
+    assert phase_index(PHASE_SYNTHESIS) == TOTAL_PHASES - 1
+
+
+def test_phase_index_unknown_falls_back_to_zero():
+    assert phase_index("not-a-phase") == 0
+    assert phase_index("") == 0
+
+
+def test_every_phase_has_a_label():
+    for phase in PHASE_ORDER:
+        assert PHASE_LABELS.get(phase)
@@ -92,6 +92,56 @@ def test_worker_run_produces_report(monkeypatch):
        VectorStore("ds_worker_test_one").drop()


+def test_index_chunks_emits_batch_progress(monkeypatch):
+    async def fake_embed_async(texts, api_key, **kwargs):
+        return local_embed(texts)
+
+    monkeypatch.setattr(worker_module, "embed_texts", fake_embed_async)
+    monkeypatch.setattr(worker_module, "EMBED_BATCH", 1)
+
+    frames = []
+    pages = [
+        CrawledPage(
+            url="https://example.com/a",
+            title="Page A",
+            text="The transistor was invented at Bell Labs. " * 30,
+            source="httpx",
+            status=200,
+        )
+    ]
+    store = VectorStore("ds_index_chunks_test")
+    try:
+        chunk_count, backend = run_async(
+            worker_module._index_chunks(store, pages, "k", frames.append)
+        )
+        assert chunk_count > 0
+        assert backend in ("gateway", "local")
+        batch_frames = [f for f in frames if f.get("type") == "embed_batch"]
+        assert batch_frames
+        assert all(f["total"] == chunk_count for f in batch_frames)
+        assert max(f["done"] for f in batch_frames) == chunk_count
+        done_frames = [f for f in frames if f.get("type") == "embed_done"]
+        assert done_frames and done_frames[-1]["chunk_count"] == chunk_count
+    finally:
+        store.drop()
+
+
+def test_index_chunks_empty_pages_emits_done(monkeypatch):
+    frames = []
+    store = VectorStore("ds_index_chunks_empty_test")
+    try:
+        chunk_count, backend = run_async(
+            worker_module._index_chunks(store, [], "k", frames.append)
+        )
+        assert chunk_count == 0
+        assert backend == "empty"
+        assert any(
+            f.get("type") == "embed_done" and f.get("chunk_count") == 0 for f in frames
+        )
+    finally:
+        store.drop()
+
+
 def test_worker_control_cancel_stops(monkeypatch):
    pages = [
        CrawledPage(url="https://x.example", title="X", text="content " * 40, source="httpx", status=200)