feat: add deepsearch research system with CLI prune/clear and database schema

Implement a multi-agent deep web research subsystem including CLI commands for pruning expired jobs and clearing all artifacts, database tables for sessions/messages/URL cache with indexes, config paths for chroma storage, and internal embed URL for vector operations.

tests/api/tools/deepsearch/__init__.py

@@ -0,0 +1 @@
+# retoor <retoor@molodetz.nl>

tests/api/tools/deepsearch/index.py

@@ -0,0 +1,119 @@
+# retoor <retoor@molodetz.nl>
+
+import requests
+from tests.conftest import BASE_URL
+from devplacepy.database import get_table, refresh_snapshot
+from devplacepy.services.jobs import queue
+
+
+def _json_headers():
+    return {"Accept": "application/json"}
+
+
+def _clear_deepsearch_jobs():
+    refresh_snapshot()
+    jobs = get_table("jobs")
+    for row in list(jobs.find(kind="deepsearch")):
+        jobs.delete(uid=row["uid"])
+    sessions = get_table("deepsearch_sessions")
+    for row in list(sessions.find()):
+        sessions.delete(uid=row["uid"])
+
+
+def test_deepsearch_page_renders(app_server):
+    r = requests.get(f"{BASE_URL}/tools/deepsearch")
+    assert r.status_code == 200
+    assert "DeepSearch" in r.text
+    assert "data-deepsearch-form" in r.text
+
+
+def test_run_enqueues_and_returns_uid(app_server):
+    try:
+        r = requests.post(
+            f"{BASE_URL}/tools/deepsearch/run",
+            headers=_json_headers(),
+            data={"query": "history of the transistor", "depth": "2", "max_pages": "5"},
+        )
+        assert r.status_code == 200, r.text
+        body = r.json()
+        uid = body["uid"]
+        assert uid
+        assert body["status_url"] == f"/tools/deepsearch/{uid}"
+        assert body["ws_url"] == f"/tools/deepsearch/{uid}/ws"
+
+        refresh_snapshot()
+        job = queue.get_job(uid)
+        assert job is not None
+        assert job["kind"] == "deepsearch"
+        assert job["status"] == queue.PENDING
+        assert job["payload"]["query"] == "history of the transistor"
+        assert job["payload"]["max_pages"] == 5
+        assert job["payload"]["api_key"]
+
+        session = get_table("deepsearch_sessions").find_one(uid=uid)
+        assert session is not None
+        assert session["status"] == "pending"
+    finally:
+        _clear_deepsearch_jobs()
+
+
+def test_run_rejects_blank_query(app_server):
+    try:
+        r = requests.post(
+            f"{BASE_URL}/tools/deepsearch/run",
+            headers=_json_headers(),
+            data={"query": "  ", "depth": "2"},
+            allow_redirects=False,
+        )
+        assert r.status_code in (400, 422), r.text
+    finally:
+        _clear_deepsearch_jobs()
+
+
+def test_run_rejects_depth_above_cap(app_server):
+    try:
+        r = requests.post(
+            f"{BASE_URL}/tools/deepsearch/run",
+            headers=_json_headers(),
+            data={"query": "valid question", "depth": "99"},
+            allow_redirects=False,
+        )
+        assert r.status_code in (400, 422), r.text
+    finally:
+        _clear_deepsearch_jobs()
+
+
+def test_run_rejects_max_pages_above_cap(app_server):
+    try:
+        r = requests.post(
+            f"{BASE_URL}/tools/deepsearch/run",
+            headers=_json_headers(),
+            data={"query": "valid question", "max_pages": "9999"},
+            allow_redirects=False,
+        )
+        assert r.status_code in (400, 422), r.text
+    finally:
+        _clear_deepsearch_jobs()
+
+
+def test_run_enforces_one_active_job_per_owner(app_server):
+    try:
+        first = requests.post(
+            f"{BASE_URL}/tools/deepsearch/run",
+            headers=_json_headers(),
+            data={"query": "first question"},
+        )
+        assert first.status_code == 200, first.text
+        first_uid = first.json()["uid"]
+
+        second = requests.post(
+            f"{BASE_URL}/tools/deepsearch/run",
+            headers=_json_headers(),
+            data={"query": "second question"},
+        )
+        assert second.status_code == 429, second.text
+        error = second.json()["error"]
+        assert error["status"] == 429
+        assert error["uid"] == first_uid
+    finally:
+        _clear_deepsearch_jobs()

tests/unit/services/deepsearch/__init__.py

@@ -0,0 +1 @@
+# retoor <retoor@molodetz.nl>

tests/unit/services/jobs/deepsearch/__init__.py

@@ -0,0 +1 @@
+# retoor <retoor@molodetz.nl>

tests/unit/services/openai_gateway/gateway.py

@@ -246,6 +246,36 @@ def test_embeddings_remaps_alias_to_configured_model(local_db, monkeypatch):
     assert rt._client.calls[-1][1]["model"] == "qwen/qwen3-embedding-8b"
 
 
+def test_embeddings_success_records_ledger(local_db, monkeypatch):
+    monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeEmbedClient_openai_gateway)
+    svc = GatewayService()
+    cfg = svc.effective_config()
+    cfg["gateway_force_model"] = True
+    cfg["gateway_embed_enabled"] = True
+    cfg["gateway_embed_model"] = "qwen/qwen3-embedding-8b"
+    rt = svc.runtime()
+    before = rt.embed_calls
+    resp = run_async(
+        rt.handle_embeddings(
+            {"model": "molodetz~embed", "input": "hello"},
+            cfg,
+            ("guest", "ledger_probe"),
+            "test",
+        )
+    )
+    assert resp.status_code == 200
+    payload = json.loads(bytes(resp.body).decode())
+    assert payload["data"][0]["embedding"] == [0.1, 0.2]
+    assert rt.embed_calls == before + 1
+    row = get_table("gateway_usage_ledger").find_one(owner_id="ledger_probe")
+    assert row is not None
+    assert row["backend"] == "embed"
+    assert row["endpoint"] == "embeddings"
+    assert row["requested_model"] == "molodetz~embed"
+    assert row["model"] == "qwen/qwen3-embedding-8b"
+    assert row["success"] == 1
+
+
 def test_embeddings_disabled_returns_503(local_db, monkeypatch):
     monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeEmbedClient_openai_gateway)
     svc = GatewayService()