Compare commits

..
Author SHA1 Message Date
Typosaurus e5cb4772e0 ticket #10 attempt 2 2026-07-19 19:03:57 +00:00
Typosaurus ea943ac502 ticket #10 attempt 2 2026-07-19 18:34:58 +00:00
Typosaurus 7a93c959df ticket #10 attempt 1 2026-07-19 18:00:04 +00:00
21 changed files with 253 additions and 76 deletions
File diff suppressed because one or more lines are too long
+1
View File
@@ -0,0 +1 @@
python3
+1
View File
@@ -0,0 +1 @@
/usr/bin/python3
+1
View File
@@ -0,0 +1 @@
python3
+1
View File
@@ -0,0 +1 @@
lib
+5
View File
@@ -0,0 +1,5 @@
home = /usr/bin
include-system-site-packages = false
version = 3.11.2
executable = /usr/bin/python3.11
command = /usr/bin/python3 -m venv /workspace/repo/.venv
+2 -1
View File
@@ -16,7 +16,7 @@ from .email import EMAIL_ACCOUNT_DEFAULTS, list_email_accounts, get_email_accoun
from .notifications import NOTIFICATION_TYPES, NOTIFICATION_CHANNELS, _NOTIFICATION_CHANNEL_COLUMNS, _NOTIFICATION_CHANNEL_DEFAULTS, _NOTIFICATION_TYPE_KEYS, _notification_prefs_cache, _notification_default, get_notification_default, set_notification_default, _notification_overrides, notification_enabled, get_notification_prefs, set_notification_pref, reset_notification_prefs, mark_notifications_read_by_target
from .forks import record_fork, get_fork_parent, count_forks, soft_delete_fork_relations, delete_fork_relations
from .follows import get_follow_counts, get_follow_list, get_following_among
from .deepsearch import _ds_now, create_deepsearch_session, update_deepsearch_session, get_deepsearch_session, add_deepsearch_message, get_deepsearch_messages, get_cached_deepsearch_url, upsert_deepsearch_url_cache
from .deepsearch import _ds_now, create_deepsearch_session, update_deepsearch_session, get_deepsearch_session, add_deepsearch_message, get_deepsearch_messages, get_cached_deepsearch_url, upsert_deepsearch_url_cache, list_deepsearch_sessions
from .ranking import VOTABLE_TARGETS, STAR_TARGETS, _authors_cache, _ranked_authors, _rank_map, get_top_authors, get_leaderboard, get_user_rank, get_user_stars, clear_user_stars, update_target_stars, soft_delete_engagement, delete_engagement, get_target_owner_uid
from .comments import _drop_blocked, _build_comment_items, load_comments, get_recent_comments_by_target_uids, get_recent_comments_by_post_uids, load_comments_by_target_uids
from .content import resolve_by_slug, resolve_object_url, get_uids_by_username_match, text_search_clause, get_daily_topic, get_featured_news
@@ -182,6 +182,7 @@ __all__ = [
"get_deepsearch_messages",
"get_cached_deepsearch_url",
"upsert_deepsearch_url_cache",
"list_deepsearch_sessions",
"VOTABLE_TARGETS",
"STAR_TARGETS",
"_authors_cache",
+17
View File
@@ -84,6 +84,23 @@ def get_deepsearch_messages(session_uid: str, limit: int = 50) -> list[dict]:
)
def list_deepsearch_sessions(
owner_kind: str, owner_id: str, limit: int = 50
) -> list[dict]:
if "deepsearch_sessions" not in db.tables:
return []
rows = list(
get_table("deepsearch_sessions").find(
owner_kind=owner_kind,
owner_id=owner_id,
deleted_at=None,
order_by=["created_at desc"],
_limit=limit,
)
)
return sorted(rows, key=lambda r: (r.get("created_at", ""), r.get("uid", "")), reverse=True)
def get_cached_deepsearch_url(url_hash: str) -> dict | None:
if "deepsearch_url_cache" not in db.tables:
return None
+32 -6
View File
@@ -11,7 +11,11 @@ from devplacepy import database
from devplacepy.config import DEEPSEARCH_DIR
from devplacepy.models import DeepsearchChatForm, DeepsearchRunForm
from devplacepy.responses import respond
from devplacepy.schemas import DeepsearchJobOut, DeepsearchSessionOut
from devplacepy.schemas import (
DeepsearchJobOut,
DeepsearchListOut,
DeepsearchSessionOut,
)
from devplacepy.seo import base_seo_context, site_url, web_application_schema, website_schema
from devplacepy.services.deepsearch.chat import DeepsearchChat
from devplacepy.services.deepsearch.export import to_json, to_markdown, to_pdf
@@ -100,10 +104,13 @@ async def deepsearch_page(request: Request):
web_application_schema("DeepSearch", description, "/tools/deepsearch", base),
],
)
sessions = []
if user:
sessions = database.list_deepsearch_sessions("user", user["uid"])
return templates.TemplateResponse(
request,
"tools/deepsearch.html",
{**seo_ctx, "request": request, "user": user},
{**seo_ctx, "request": request, "user": user, "history_sessions": sessions},
)
@router.post("/run")
@@ -197,6 +204,27 @@ def _enqueue(uid: str, payload: dict, owner_kind: str, owner_id: str, query: str
}
)
@router.get("/list")
async def deepsearch_list(request: Request):
owner_kind, owner_id = owner_for(request)
sessions = database.list_deepsearch_sessions(owner_kind, owner_id)
return JSONResponse(
DeepsearchListOut.model_validate(
{"sessions": [_session_summary(s) for s in sessions]}
).model_dump(mode="json")
)
def _session_summary(session: dict) -> dict:
return {
"uid": session.get("uid", ""),
"query": session.get("query", ""),
"status": session.get("status", ""),
"created_at": session.get("created_at", ""),
}
@router.get("/{uid}")
async def deepsearch_status(request: Request, uid: str):
job = queue.get_job(uid)
@@ -227,8 +255,7 @@ def _session_context(request: Request, uid: str, job: dict, session: dict) -> di
user = get_current_user(request)
viewer_is_admin = is_admin(user)
done = bool(report) or job.get("status") == queue.DONE
cost_usd = report.get("cost_usd", 0.0) if report else 0.0
ctx = {
return {
"uid": uid,
"status": queue.DONE if done else job.get("status", ""),
"query": report.get("query") or session.get("query"),
@@ -248,7 +275,6 @@ def _session_context(request: Request, uid: str, job: dict, session: dict) -> di
"export_md_url": f"/tools/deepsearch/{uid}/export.md" if done else None,
"export_json_url": f"/tools/deepsearch/{uid}/export.json" if done else None,
"export_pdf_url": f"/tools/deepsearch/{uid}/export.pdf" if done else None,
"cost_usd": cost_usd if viewer_is_admin else None,
"viewer_is_admin": viewer_is_admin,
"viewer_owns": _owns(request, session),
"created_at": session.get("created_at"),
@@ -257,7 +283,6 @@ def _session_context(request: Request, uid: str, job: dict, session: dict) -> di
"user": user,
"meta_robots": "noindex,nofollow",
}
return ctx
@router.get("/{uid}/session")
async def deepsearch_session(request: Request, uid: str):
@@ -325,6 +350,7 @@ async def deepsearch_export_pdf(request: Request, uid: str):
raise not_found("Report not ready")
return Response(content=to_pdf(report), media_type="application/pdf")
@router.websocket("/{uid}/ws")
async def deepsearch_ws(websocket: WebSocket, uid: str):
await websocket.accept()
+2
View File
@@ -79,7 +79,9 @@ from devplacepy.schemas.containers import (
from devplacepy.schemas.jobs import (
DbQueryJobOut,
DeepsearchJobOut,
DeepsearchListOut,
DeepsearchSessionOut,
DeepsearchSessionSummary,
ForkJobOut,
PlanningJobOut,
SeoJobOut,
+11 -1
View File
@@ -136,13 +136,23 @@ class DeepsearchSessionOut(_Out):
export_md_url: Optional[str] = None
export_json_url: Optional[str] = None
export_pdf_url: Optional[str] = None
cost_usd: Optional[float] = None
viewer_is_admin: bool = False
viewer_owns: bool = False
created_at: Optional[str] = None
completed_at: Optional[str] = None
class DeepsearchSessionSummary(_Out):
uid: str = ""
query: str = ""
status: str = ""
created_at: str = ""
class DeepsearchListOut(_Out):
sessions: list = []
class DbQueryJobOut(_Out):
uid: str = ""
kind: str = ""
+1 -5
View File
@@ -7,7 +7,6 @@ import time
from devplacepy import stealth
from devplacepy.config import INTERNAL_GATEWAY_URL, INTERNAL_MODEL
from devplacepy.services.openai_gateway.usage import parse_usage_headers
logger = logging.getLogger(__name__)
@@ -42,10 +41,7 @@ async def request_completion(
if response.status_code >= 400:
raise RuntimeError(f"chat gateway returned {response.status_code}")
data: dict = response.json()
cost_info = parse_usage_headers(response.headers) or {}
usage = data.get("usage") or {}
usage["cost_usd"] = cost_info.get("cost_usd", 0.0)
return data, usage, elapsed_ms
return data, data.get("usage") or {}, elapsed_ms
async def complete_chat(
@@ -108,6 +108,18 @@ TOOLS_ACTIONS: tuple[Action, ...] = (
params=(path("uid", "DeepSearch job uid returned by deepsearch."),),
requires_auth=False,
),
Action(
name="deepsearch_list",
method="GET",
path="/tools/deepsearch/list",
summary="List the user's past DeepSearch sessions",
description=(
"Returns the signed-in user's deep research session history, newest first, each "
"with its uid, query, status and created_at."
),
params=(),
requires_auth=True,
),
Action(
name="isslop",
method="POST",
@@ -5,11 +5,10 @@ from __future__ import annotations
import json
import logging
import re
from typing import Callable, Tuple
from typing import Callable
from devplacepy import stealth
from devplacepy.config import INTERNAL_GATEWAY_URL, INTERNAL_MODEL
from devplacepy.services.openai_gateway.usage import parse_usage_headers
from .phases import PHASE_PLANNING
@@ -62,7 +61,7 @@ def _noop(frame: dict) -> None:
async def plan_queries(
query: str, api_key: str, emit: Callable[[dict], None] = _noop
) -> Tuple[list[str], float]:
) -> list[str]:
emit(
{
"type": "substep",
@@ -90,8 +89,6 @@ async def plan_queries(
)
if response.status_code >= 400:
raise RuntimeError(f"planner gateway returned {response.status_code}")
cost_info = parse_usage_headers(response.headers) or {}
planner_cost = cost_info.get("cost_usd", 0.0)
data = response.json()
content = (
(data.get("choices") or [{}])[0].get("message", {}).get("content") or ""
@@ -105,7 +102,7 @@ async def plan_queries(
"message": f"Planned {len(parsed)} search angles",
}
)
return parsed, planner_cost
return parsed
except Exception as exc:
logger.warning("deepsearch query planner failed, using fallback: %s", exc)
fallback = _fallback(query)
@@ -116,4 +113,4 @@ async def plan_queries(
"message": f"Planner unavailable, using {len(fallback)} heuristic angles",
}
)
return fallback, 0.0
return fallback
@@ -40,7 +40,6 @@ class Orchestration:
source_diversity: float = 0.0
score: int = 0
synthesis: str = "agents"
total_cost_usd: float = 0.0
def _domain(url: str) -> str:
@@ -166,7 +165,6 @@ async def _complete(
"tokens_in": int(raw_usage.get("prompt_tokens") or prompt_chars // 4),
"tokens_out": int(raw_usage.get("completion_tokens") or len(text) // 4),
"elapsed_ms": elapsed_ms,
"cost_usd": float(raw_usage.get("cost_usd", 0.0)),
}
return text, usage
@@ -298,7 +296,6 @@ def _agent_done(emit: Callable[[dict], None], agent: str, usage: dict) -> None:
"elapsed_ms": usage.get("elapsed_ms", 0),
"tokens_in": usage.get("tokens_in", 0),
"tokens_out": usage.get("tokens_out", 0),
"cost_usd": usage.get("cost_usd", 0.0),
}
)
@@ -349,7 +346,6 @@ async def orchestrate(
emit: Callable[[dict], None],
store=None,
queries: list[str] | None = None,
planner_cost_usd: float = 0.0,
) -> Orchestration:
diversity = source_diversity(pages)
if not pages:
@@ -422,12 +418,6 @@ async def orchestrate(
score = int(
min(SCORE_MAX, (confidence * 0.5 + diversity * 0.3 + coverage * 0.2) * SCORE_MAX)
)
total_cost = (
planner_cost_usd
+ float(summary_usage.get("cost_usd", 0.0))
+ float(findings_usage.get("cost_usd", 0.0))
+ float(linker_usage.get("cost_usd", 0.0))
)
return Orchestration(
summary=summary,
findings=findings,
@@ -435,7 +425,6 @@ async def orchestrate(
source_diversity=diversity,
score=score,
synthesis="agents",
total_cost_usd=total_cost,
)
except Exception as exc:
logger.warning("deepsearch orchestration failed, using heuristic: %s", exc)
@@ -165,7 +165,7 @@ async def _run(payload: dict, output_dir: Path) -> dict:
should_stop = _make_stop(output_dir)
_stage("planning", "Planning research queries", PHASE_PLANNING)
queries, planner_cost = await plan_queries(query, api_key, _emit)
queries = await plan_queries(query, api_key, _emit)
_emit({"type": "queries", "queries": queries})
_stage("searching", "Searching the web", PHASE_SEARCHING)
@@ -203,8 +203,7 @@ async def _run(payload: dict, output_dir: Path) -> dict:
_stage("analysis", "Running research agents", PHASE_ANALYSIS)
result = await orchestrate(
query, outcome.pages, api_key, _emit, store=store, queries=queries,
planner_cost_usd=planner_cost,
query, outcome.pages, api_key, _emit, store=store, queries=queries
)
_stage("synthesis", "Compiling cited report", PHASE_SYNTHESIS)
@@ -229,7 +228,6 @@ async def _run(payload: dict, output_dir: Path) -> dict:
"chunk_count": chunk_count,
"embed_backend": embed_backend,
"collection": collection,
"cost_usd": result.total_cost_usd,
}
(output_dir / "report.json").write_text(
json.dumps(report, ensure_ascii=False), encoding="utf-8"
@@ -246,7 +244,6 @@ async def _run(payload: dict, output_dir: Path) -> dict:
"synthesis": report["synthesis"],
"page_count": report["page_count"],
"chunk_count": report["chunk_count"],
"cost_usd": report["cost_usd"],
}
)
return report
@@ -95,6 +95,22 @@
<p>Research complete.</p>
<a class="btn btn-primary" data-deepsearch-open href="#">Open report and chat</a>
</section>
{% if user and history_sessions %}
<section class="ds-history mt-6">
<h2 class="text-lg font-semibold mb-2">My Research History</h2>
<ul class="space-y-1">
{% for session in history_sessions %}
<li>
<a href="/tools/deepsearch/{{ session.uid }}/session"
class="text-blue-600 hover:underline text-sm">
{{ session.query }} — <span class="text-gray-500">{{ session.created_at[:16].replace('T', ' ') }}</span>
</a>
</li>
{% endfor %}
</ul>
</section>
{% endif %}
</div>
</div>
{% endblock %}
@@ -16,9 +16,6 @@
<span class="ds-metric"><strong>{{ source_diversity }}</strong> diversity</span>
<span class="ds-metric"><strong>{{ page_count }}</strong> sources</span>
<span class="ds-metric"><strong>{{ chunk_count }}</strong> chunks</span>
{% if viewer_is_admin and cost_usd is not none %}
<span class="ds-metric"><strong>${{ "%.6f"|format(cost_usd) }}</strong> cost</span>
{% endif %}
</div>
{% if export_md_url %}
<div class="ds-exports">
+16 -8
View File
@@ -1,8 +1,16 @@
2026-07-19T17:31:36 INFO logging initialised at /workspace/repo/dpc.log
2026-07-19T17:31:36 DEBUG model=molodetz-pro fps=30
2026-07-19T17:31:36 INFO read task from file: /workspace/prompts/research-10.txt
2026-07-19T17:31:36 INFO settings merged: model=<default> allow=0 deny=0 ask=0
2026-07-19T19:05:24 INFO logging initialised at /workspace/repo/dpc.log
2026-07-19T19:05:24 DEBUG model=molodetz-pro fps=30
2026-07-19T19:05:24 INFO read task from file: /workspace/prompts/execution-1.txt
2026-07-19T19:05:24 INFO settings merged: model=<default> allow=0 deny=0 ask=0
2026-07-19T17:32:58 INFO logging initialised at /workspace/repo/dpc.log
2026-07-19T17:32:58 DEBUG model=molodetz-pro fps=30
2026-07-19T17:32:58 INFO read task from file: /workspace/prompts/research-9.txt
2026-07-19T17:32:58 INFO settings merged: model=<default> allow=0 deny=0 ask=0
2026-07-19T17:38:24 INFO logging initialised at /workspace/repo/dpc.log
2026-07-19T17:38:24 DEBUG model=molodetz-pro fps=30
2026-07-19T17:38:24 INFO read task from file: /workspace/prompts/execution-1.txt
2026-07-19T17:38:24 INFO settings merged: model=<default> allow=0 deny=0 ask=0
2026-07-19T18:33:02 INFO logging initialised at /workspace/repo/dpc.log
2026-07-19T18:33:02 DEBUG model=molodetz-pro fps=30
2026-07-19T18:33:02 INFO read task from file: /workspace/prompts/execution-2.txt
2026-07-19T18:33:02 INFO settings merged: model=<default> allow=0 deny=0 ask=0
2026-07-19T19:00:28 INFO logging initialised at /workspace/repo/dpc.log
2026-07-19T19:00:28 DEBUG model=molodetz-pro fps=30
2026-07-19T19:00:28 INFO read task from file: /workspace/prompts/execution-3.txt
2026-07-19T19:00:28 INFO settings merged: model=<default> allow=0 deny=0 ask=0
+129
View File
@@ -0,0 +1,129 @@
# retoor <retoor@molodetz.nl>
import time
import pytest
import requests
from tests.conftest import BASE_URL
from devplacepy.database import (
create_deepsearch_session,
get_table,
refresh_snapshot,
set_setting,
)
from devplacepy.services.jobs import queue
JSON = {"Accept": "application/json"}
_counter = [0]
@pytest.fixture(scope="module", autouse=True)
def _settings(app_server):
set_setting("rate_limit_per_minute", "1000000")
set_setting("registration_open", "1")
yield
def _user():
_counter[0] += 1
name = f"dsl{int(time.time() * 1000)}{_counter[0]}"
requests.post(
f"{BASE_URL}/auth/signup",
data={
"username": name,
"email": f"{name}@t.dev",
"password": "secret123",
"confirm_password": "secret123",
},
allow_redirects=True,
)
refresh_snapshot()
user = get_table("users").find_one(username=name)
return user["uid"], user["api_key"]
def _seed_session(owner_id, query="test query"):
uid = queue.enqueue(
"deepsearch",
{"query": query, "depth": 1, "max_pages": 5},
"user",
owner_id,
"DeepSearch",
)
create_deepsearch_session(
uid, "user", owner_id, query, 1, 5, f"ds_{uid.replace('-', '')}"
)
refresh_snapshot()
return uid
def _clear():
refresh_snapshot()
jobs = get_table("jobs")
for row in list(jobs.find(kind="deepsearch")):
jobs.delete(uid=row["uid"])
sessions = get_table("deepsearch_sessions")
for row in list(sessions.find()):
sessions.delete(uid=row["uid"])
def test_list_returns_own_sessions(app_server):
try:
owner_uid, api_key = _user()
uid = _seed_session(owner_uid, query="first research")
headers = {"X-API-KEY": api_key, **JSON}
r = requests.get(f"{BASE_URL}/tools/deepsearch/list", headers=headers)
assert r.status_code == 200, r.text
body = r.json()
assert "sessions" in body
assert len(body["sessions"]) == 1
assert body["sessions"][0]["uid"] == uid
assert body["sessions"][0]["query"] == "first research"
assert body["sessions"][0]["status"] == "pending"
assert "created_at" in body["sessions"][0]
finally:
_clear()
def test_list_multiple_sessions_sorted_newest_first(app_server):
try:
owner_uid, api_key = _user()
uids = []
for i in range(3):
uids.append(_seed_session(owner_uid, query=f"research {i}"))
headers = {"X-API-KEY": api_key, **JSON}
r = requests.get(f"{BASE_URL}/tools/deepsearch/list", headers=headers)
assert r.status_code == 200, r.text
body = r.json()
assert len(body["sessions"]) == 3
uids_in_response = [s["uid"] for s in body["sessions"]]
assert sorted(uids_in_response, reverse=True) == uids_in_response
actual_queries = [s["query"] for s in body["sessions"]]
assert len(set(actual_queries)) == 3
finally:
_clear()
def test_list_does_not_include_other_users_sessions(app_server):
try:
owner_uid, api_key = _user()
_seed_session(owner_uid, query="my research")
other_uid, _ = _user()
_seed_session(other_uid, query="other research")
headers = {"X-API-KEY": api_key, **JSON}
r = requests.get(f"{BASE_URL}/tools/deepsearch/list", headers=headers)
assert r.status_code == 200, r.text
body = r.json()
assert len(body["sessions"]) == 1
assert body["sessions"][0]["query"] == "my research"
finally:
_clear()
def test_list_empty_for_no_sessions(app_server):
owner_uid, api_key = _user()
headers = {"X-API-KEY": api_key, **JSON}
r = requests.get(f"{BASE_URL}/tools/deepsearch/list", headers=headers)
assert r.status_code == 200, r.text
body = r.json()
assert "sessions" in body
assert len(body["sessions"]) == 0
-28
View File
@@ -39,7 +39,6 @@ def _seed_done_session(owner_id="ds-session-owner"):
"source_diversity": 0.5,
"page_count": 3,
"chunk_count": 12,
"cost_usd": 0.123456,
}
result = {
"query": "the question",
@@ -48,7 +47,6 @@ def _seed_done_session(owner_id="ds-session-owner"):
"source_diversity": 0.5,
"page_count": 3,
"chunk_count": 12,
"cost_usd": 0.123456,
"report": report,
}
get_table("jobs").update(
@@ -99,7 +97,6 @@ def test_session_json_shape(app_server):
assert body["sources"]
assert "viewer_is_admin" in body
assert "viewer_owns" in body
assert body.get("cost_usd") is None # guest user, not admin
finally:
_clear()
@@ -111,7 +108,6 @@ def test_session_html_renders_without_jinja_global_collision(app_server):
assert r.status_code == 200, r.text
assert "the question" in r.text
assert "dp-deepsearch-chat" in r.text
assert "0.123456" not in r.text # guest user, cost hidden
finally:
_clear()
@@ -146,7 +142,6 @@ def test_session_reads_disk_report_before_result_commit(app_server):
"synthesis": "agents",
"page_count": 12,
"chunk_count": 61,
"cost_usd": 0.123456,
}
session_dir = DEEPSEARCH_DIR / uid
session_dir.mkdir(parents=True, exist_ok=True)
@@ -208,26 +203,3 @@ def test_export_json(app_server):
def test_export_unknown_uid_404(app_server):
r = requests.get(f"{BASE_URL}/tools/deepsearch/nope/export.md")
assert r.status_code == 404
def test_session_admin_sees_cost(app_server, seeded_db):
try:
uid = _seed_done_session("alice_test")
s = requests.Session()
creds = seeded_db["alice"]
s.post(
f"{BASE_URL}/auth/login",
data={"email": creds["email"], "password": creds["password"]},
allow_redirects=True,
)
r = s.get(f"{BASE_URL}/tools/deepsearch/{uid}/session", headers=_json_headers())
assert r.status_code == 200, r.text
body = r.json()
assert body.get("cost_usd") == 0.123456
html = s.get(f"{BASE_URL}/tools/deepsearch/{uid}/session")
assert html.status_code == 200, html.text
assert "0.123456" in html.text
assert "cost" in html.text
finally:
_clear()