feat: add user_id index to profiles table for faster lookups

The index on user_id column in profiles table improves query performance for user-specific operations, reducing full table scans during authentication and profile retrieval.

tests/api/tools/deepsearch/session.py

@@ -0,0 +1,148 @@
+# retoor <retoor@molodetz.nl>
+
+import json
+
+import requests
+from tests.conftest import BASE_URL
+from devplacepy.database import (
+    create_deepsearch_session,
+    get_table,
+    refresh_snapshot,
+)
+from devplacepy.services.jobs import queue
+
+
+def _json_headers():
+    return {"Accept": "application/json"}
+
+
+def _seed_done_session(owner_id="ds-session-owner"):
+    uid = queue.enqueue(
+        "deepsearch",
+        {"query": "the question", "depth": 2, "max_pages": 10},
+        "user",
+        owner_id,
+        "DeepSearch: the question",
+    )
+    create_deepsearch_session(
+        uid, "user", owner_id, "the question", 2, 10, f"ds_{uid.replace('-', '')}"
+    )
+    report = {
+        "query": "the question",
+        "summary": "A grounded summary.",
+        "findings": [
+            {"title": "Finding one", "detail": "Detail.", "confidence": 0.8, "citations": [1]}
+        ],
+        "gaps": ["An open gap."],
+        "sources": [{"url": "https://example.com", "title": "Example", "source": "httpx"}],
+        "score": 70,
+        "confidence": 0.7,
+        "source_diversity": 0.5,
+        "page_count": 3,
+        "chunk_count": 12,
+    }
+    result = {
+        "query": "the question",
+        "score": 70,
+        "confidence": 0.7,
+        "source_diversity": 0.5,
+        "page_count": 3,
+        "chunk_count": 12,
+        "report": report,
+    }
+    get_table("jobs").update(
+        {"uid": uid, "status": queue.DONE, "result": json.dumps(result)}, ["uid"]
+    )
+    get_table("deepsearch_sessions").update(
+        {
+            "uid": uid,
+            "status": "done",
+            "score": 70,
+            "confidence": 0.7,
+            "source_diversity": 0.5,
+            "page_count": 3,
+            "chunk_count": 12,
+            "summary": "A grounded summary.",
+        },
+        ["uid"],
+    )
+    refresh_snapshot()
+    return uid
+
+
+def _clear():
+    refresh_snapshot()
+    jobs = get_table("jobs")
+    for row in list(jobs.find(kind="deepsearch")):
+        jobs.delete(uid=row["uid"])
+    sessions = get_table("deepsearch_sessions")
+    for row in list(sessions.find()):
+        sessions.delete(uid=row["uid"])
+
+
+def test_session_json_shape(app_server):
+    try:
+        uid = _seed_done_session()
+        r = requests.get(
+            f"{BASE_URL}/tools/deepsearch/{uid}/session", headers=_json_headers()
+        )
+        assert r.status_code == 200, r.text
+        body = r.json()
+        assert body["uid"] == uid
+        assert body["status"] == "done"
+        assert body["query"] == "the question"
+        assert body["score"] == 70
+        assert body["chat_ws_url"] == f"/tools/deepsearch/{uid}/chat"
+        assert body["export_md_url"] == f"/tools/deepsearch/{uid}/export.md"
+        assert body["findings"]
+        assert body["sources"]
+        assert "viewer_is_admin" in body
+        assert "viewer_owns" in body
+    finally:
+        _clear()
+
+
+def test_session_html_renders_without_jinja_global_collision(app_server):
+    try:
+        uid = _seed_done_session()
+        r = requests.get(f"{BASE_URL}/tools/deepsearch/{uid}/session")
+        assert r.status_code == 200, r.text
+        assert "the question" in r.text
+        assert "dp-deepsearch-chat" in r.text
+    finally:
+        _clear()
+
+
+def test_session_unknown_uid_404(app_server):
+    r = requests.get(
+        f"{BASE_URL}/tools/deepsearch/nope/session", headers=_json_headers()
+    )
+    assert r.status_code == 404
+
+
+def test_export_markdown(app_server):
+    try:
+        uid = _seed_done_session()
+        r = requests.get(f"{BASE_URL}/tools/deepsearch/{uid}/export.md")
+        assert r.status_code == 200, r.text
+        assert "DeepSearch report" in r.text
+        assert "Finding one" in r.text
+    finally:
+        _clear()
+
+
+def test_export_json(app_server):
+    try:
+        uid = _seed_done_session()
+        r = requests.get(f"{BASE_URL}/tools/deepsearch/{uid}/export.json")
+        assert r.status_code == 200, r.text
+        body = r.json()
+        assert body["query"] == "the question"
+        assert body["findings"]
+    finally:
+        _clear()
+
+
+def test_export_unknown_uid_404(app_server):
+    r = requests.get(f"{BASE_URL}/tools/deepsearch/nope/export.md")
+    assert r.status_code == 404

tests/api/tools/deepsearch/status.py

@@ -0,0 +1,63 @@
+# retoor <retoor@molodetz.nl>
+
+import requests
+from tests.conftest import BASE_URL
+from devplacepy.database import get_table, refresh_snapshot
+from devplacepy.services.jobs import queue
+
+
+def _json_headers():
+    return {"Accept": "application/json"}
+
+
+def _clear():
+    refresh_snapshot()
+    jobs = get_table("jobs")
+    for row in list(jobs.find(kind="deepsearch")):
+        jobs.delete(uid=row["uid"])
+
+
+def test_status_shape_for_pending_job(app_server):
+    try:
+        uid = queue.enqueue(
+            "deepsearch",
+            {"query": "a research question", "depth": 2, "max_pages": 10},
+            "user",
+            "ds-status-owner",
+            "DeepSearch: a research question",
+        )
+        refresh_snapshot()
+        r = requests.get(
+            f"{BASE_URL}/tools/deepsearch/{uid}", headers=_json_headers()
+        )
+        assert r.status_code == 200, r.text
+        body = r.json()
+        assert body["uid"] == uid
+        assert body["kind"] == "deepsearch"
+        assert body["status"] == "pending"
+        assert body["query"] == "a research question"
+        assert body["ws_url"] == f"/tools/deepsearch/{uid}/ws"
+        assert body["session_url"] is None
+        assert body["score"] is None
+        assert body["page_count"] == 0
+    finally:
+        _clear()
+
+
+def test_status_unknown_uid_404(app_server):
+    r = requests.get(
+        f"{BASE_URL}/tools/deepsearch/nope-not-a-job", headers=_json_headers()
+    )
+    assert r.status_code == 404
+
+
+def test_status_rejects_non_deepsearch_kind(app_server):
+    try:
+        uid = queue.enqueue("other", {}, "user", "ds-kind-owner", "x")
+        refresh_snapshot()
+        r = requests.get(
+            f"{BASE_URL}/tools/deepsearch/{uid}", headers=_json_headers()
+        )
+        assert r.status_code == 404
+    finally:
+        get_table("jobs").delete(kind="other")

tests/unit/services/deepsearch/chat.py

@@ -0,0 +1,55 @@
+# retoor <retoor@molodetz.nl>
+
+import asyncio
+
+from devplacepy.services.deepsearch import chat as chat_module
+from devplacepy.services.deepsearch.chat import DeepsearchChat
+from devplacepy.services.deepsearch.embeddings import local_embed
+from devplacepy.services.deepsearch.store import Chunk, VectorStore
+
+
+def _seed(collection):
+    store = VectorStore(collection)
+    chunks = [
+        Chunk(uid="c0", text="The transistor was invented at Bell Labs.", url="https://a.example", title="A"),
+        Chunk(uid="c1", text="Silicon wafers are used to make chips.", url="https://b.example", title="B"),
+    ]
+    vectors = local_embed([c.text for c in chunks]).vectors
+    store.add(chunks, vectors)
+
+
+def test_answer_is_grounded_and_cited(monkeypatch):
+    collection = "ds_chat_test_grounded"
+    _seed(collection)
+    try:
+        async def fake_embed(texts, api_key, **kwargs):
+            return local_embed(texts)
+
+        async def fake_complete(messages, api_key, **kwargs):
+            return "The transistor was invented at Bell Labs [1]."
+
+        monkeypatch.setattr(chat_module, "embed_texts", fake_embed)
+        monkeypatch.setattr(chat_module, "complete_chat", fake_complete)
+
+        chat = DeepsearchChat(collection, "k")
+        answer = asyncio.run(chat.answer("where was the transistor invented"))
+        assert "Bell Labs" in answer.text
+        assert answer.citations
+        assert answer.citations[0]["url"].startswith("https://")
+    finally:
+        VectorStore(collection).drop()
+
+
+def test_answer_when_no_chunks(monkeypatch):
+    collection = "ds_chat_test_empty"
+    try:
+        async def fake_embed(texts, api_key, **kwargs):
+            return local_embed(texts)
+
+        monkeypatch.setattr(chat_module, "embed_texts", fake_embed)
+        chat = DeepsearchChat(collection, "k")
+        answer = asyncio.run(chat.answer("anything"))
+        assert answer.citations == []
+        assert "did not capture" in answer.text
+    finally:
+        VectorStore(collection).drop()

tests/unit/services/deepsearch/store.py

@@ -0,0 +1,52 @@
+# retoor <retoor@molodetz.nl>
+
+from devplacepy.services.deepsearch.embeddings import local_embed
+from devplacepy.services.deepsearch.store import Chunk, VectorStore
+
+
+def _chunks():
+    texts = [
+        "The transistor was invented at Bell Labs in nineteen forty seven.",
+        "Silicon is the primary semiconductor material used in chips.",
+        "Quantum tunnelling limits how small a transistor can become.",
+    ]
+    chunks = [
+        Chunk(uid=f"c{i}", text=text, url=f"https://s{i}.example", title=f"Source {i}")
+        for i, text in enumerate(texts)
+    ]
+    return chunks
+
+
+def test_add_and_count():
+    store = VectorStore("ds_store_test_count")
+    try:
+        chunks = _chunks()
+        vectors = local_embed([c.text for c in chunks]).vectors
+        store.add(chunks, vectors)
+        assert store.count() == 3
+    finally:
+        store.drop()
+
+
+def test_hybrid_search_returns_relevant_chunk():
+    store = VectorStore("ds_store_test_hybrid")
+    try:
+        chunks = _chunks()
+        vectors = local_embed([c.text for c in chunks]).vectors
+        store.add(chunks, vectors)
+        query = "where was the transistor invented"
+        query_vector = local_embed([query]).vectors[0]
+        results = store.hybrid_search(query, query_vector, top_k=2)
+        assert results
+        assert any("transistor" in r.text.lower() for r in results)
+    finally:
+        store.drop()
+
+
+def test_keyword_scores_rank_match_higher():
+    store = VectorStore("ds_store_test_keyword")
+    chunks = _chunks()
+    scores = store.keyword_scores("silicon semiconductor", chunks)
+    assert scores
+    best = max(scores, key=scores.get)
+    assert "silicon" in next(c.text for c in chunks if c.uid == best).lower()

tests/unit/services/jobs/deepsearch/worker.py

@@ -0,0 +1,103 @@
+# retoor <retoor@molodetz.nl>
+
+import asyncio
+import json
+import tempfile
+from pathlib import Path
+
+from devplacepy.services.jobs.deepsearch import crawl as crawl_module
+from devplacepy.services.jobs.deepsearch import worker as worker_module
+from devplacepy.services.jobs.deepsearch.crawl import CrawledPage, CrawlOutcome
+from devplacepy.services.deepsearch.embeddings import local_embed
+from devplacepy.services.deepsearch.store import VectorStore
+
+
+def _patch_pipeline(monkeypatch, pages):
+    async def fake_plan(query, api_key):
+        return [query, f"{query} overview"]
+
+    async def fake_search(queries):
+        return [{"url": page.url, "title": page.title, "description": ""} for page in pages]
+
+    async def fake_crawl(candidates, max_pages, emit, is_cached, should_stop):
+        outcome = CrawlOutcome()
+        for page in pages[:max_pages]:
+            emit({"type": "page_loaded", "url": page.url, "done": 1, "total": len(pages)})
+            outcome.pages.append(page)
+        return outcome
+
+    def fake_embed(texts, api_key):
+        return local_embed(texts)
+
+    async def fake_embed_async(texts, api_key, **kwargs):
+        return local_embed(texts)
+
+    async def fake_orchestrate(question, crawled, api_key, emit):
+        from devplacepy.services.jobs.deepsearch.orchestrate import Orchestration
+
+        return Orchestration(
+            summary="A summary.",
+            findings=[{"title": "F", "detail": "D", "confidence": 0.5, "citations": [1]}],
+            gaps=["gap"],
+            confidence=0.6,
+            source_diversity=0.5,
+            score=42,
+        )
+
+    monkeypatch.setattr(worker_module, "plan_queries", fake_plan)
+    monkeypatch.setattr(worker_module, "search_queries", fake_search)
+    monkeypatch.setattr(worker_module, "crawl", fake_crawl)
+    monkeypatch.setattr(worker_module, "embed_texts", fake_embed_async)
+    monkeypatch.setattr(worker_module, "orchestrate", fake_orchestrate)
+
+
+def test_worker_run_produces_report(monkeypatch):
+    pages = [
+        CrawledPage(
+            url="https://example.com/a",
+            title="Page A",
+            text="The transistor was invented at Bell Labs. " * 20,
+            source="httpx",
+            status=200,
+        ),
+        CrawledPage(
+            url="https://other.example/b",
+            title="Page B",
+            text="Semiconductors are made from silicon. " * 20,
+            source="playwright",
+            status=200,
+        ),
+    ]
+    _patch_pipeline(monkeypatch, pages)
+    with tempfile.TemporaryDirectory() as tmp:
+        output_dir = Path(tmp)
+        payload = {
+            "query": "history of the transistor",
+            "max_pages": 5,
+            "depth": 2,
+            "api_key": "k",
+            "collection": "ds_worker_test_one",
+            "cached_hashes": [],
+        }
+        report = asyncio.run(worker_module._run(payload, output_dir))
+        assert report["query"] == "history of the transistor"
+        assert report["page_count"] == 2
+        assert report["chunk_count"] > 0
+        assert report["summary"] == "A summary."
+        assert report["findings"]
+        assert (output_dir / "report.json").is_file()
+        cache = json.loads((output_dir / "url_cache.json").read_text())
+        assert len(cache) == 2
+        VectorStore("ds_worker_test_one").drop()
+
+
+def test_worker_control_cancel_stops(monkeypatch):
+    pages = [
+        CrawledPage(url="https://x.example", title="X", text="content " * 40, source="httpx", status=200)
+    ]
+    _patch_pipeline(monkeypatch, pages)
+    with tempfile.TemporaryDirectory() as tmp:
+        output_dir = Path(tmp)
+        (output_dir / "control.json").write_text(json.dumps({"state": "cancelled"}))
+        should_stop = worker_module._make_stop(output_dir)
+        assert asyncio.run(should_stop()) is True