forked from retoor/devplacepy
feat: add deepsearch research system with CLI prune/clear and database schema
Implement a multi-agent deep web research subsystem including CLI commands for pruning expired jobs and clearing all artifacts, database tables for sessions/messages/URL cache with indexes, config paths for chroma storage, and internal embed URL for vector operations.
This commit is contained in:
@@ -324,6 +324,58 @@ def cmd_seo_clear(args):
|
||||
print(f"Cleared {len(jobs)} SEO audit(s) and their reports")
|
||||
|
||||
|
||||
def _remove_deepsearch_artifacts(job):
|
||||
import shutil
|
||||
from devplacepy.config import DEEPSEARCH_DIR
|
||||
from devplacepy.services.deepsearch.store import VectorStore
|
||||
|
||||
uid = job["uid"]
|
||||
collection = f"ds_{uid.replace('-', '')}"
|
||||
VectorStore(collection).drop()
|
||||
shutil.rmtree(DEEPSEARCH_DIR / uid, ignore_errors=True)
|
||||
|
||||
|
||||
def cmd_deepsearch_prune(args):
|
||||
from datetime import datetime, timezone
|
||||
from devplacepy.services.jobs import queue
|
||||
|
||||
now = datetime.now(timezone.utc)
|
||||
removed = 0
|
||||
for job in queue.list_jobs(kind="deepsearch", status=queue.DONE):
|
||||
expires_at = job.get("expires_at")
|
||||
if not expires_at:
|
||||
continue
|
||||
try:
|
||||
expiry = datetime.fromisoformat(expires_at)
|
||||
except (ValueError, TypeError):
|
||||
continue
|
||||
if expiry < now:
|
||||
_remove_deepsearch_artifacts(job)
|
||||
get_table("jobs").delete(uid=job["uid"])
|
||||
removed += 1
|
||||
_audit_cli(
|
||||
"cli.deepsearch.prune",
|
||||
f"CLI pruned {removed} expired DeepSearch jobs",
|
||||
metadata={"count": removed},
|
||||
)
|
||||
print(f"Pruned {removed} expired DeepSearch job(s)")
|
||||
|
||||
|
||||
def cmd_deepsearch_clear(args):
|
||||
from devplacepy.services.jobs import queue
|
||||
|
||||
jobs = queue.list_jobs(kind="deepsearch")
|
||||
for job in jobs:
|
||||
_remove_deepsearch_artifacts(job)
|
||||
get_table("jobs").delete(uid=job["uid"])
|
||||
_audit_cli(
|
||||
"cli.deepsearch.clear",
|
||||
f"CLI cleared all DeepSearch jobs ({len(jobs)})",
|
||||
metadata={"count": len(jobs)},
|
||||
)
|
||||
print(f"Cleared {len(jobs)} DeepSearch job(s) and their collections")
|
||||
|
||||
|
||||
def cmd_containers_list(args):
|
||||
from devplacepy.services.containers import store
|
||||
|
||||
@@ -712,6 +764,17 @@ def main():
|
||||
)
|
||||
seo_clear.set_defaults(func=cmd_seo_clear)
|
||||
|
||||
deepsearch = sub.add_parser("deepsearch", help="DeepSearch job management")
|
||||
deepsearch_sub = deepsearch.add_subparsers(title="action", dest="action")
|
||||
deepsearch_prune = deepsearch_sub.add_parser(
|
||||
"prune", help="Delete expired DeepSearch sessions and their job rows"
|
||||
)
|
||||
deepsearch_prune.set_defaults(func=cmd_deepsearch_prune)
|
||||
deepsearch_clear = deepsearch_sub.add_parser(
|
||||
"clear", help="Delete every DeepSearch session and job row"
|
||||
)
|
||||
deepsearch_clear.set_defaults(func=cmd_deepsearch_clear)
|
||||
|
||||
containers = sub.add_parser("containers", help="Container manager")
|
||||
containers_sub = containers.add_subparsers(title="action", dest="action")
|
||||
containers_sub.add_parser("list", help="List container instances").set_defaults(
|
||||
|
||||
@@ -25,6 +25,8 @@ ZIPS_DIR = DATA_DIR / "zips"
|
||||
ZIP_STAGING_DIR = DATA_DIR / "zip_staging"
|
||||
FORK_STAGING_DIR = DATA_DIR / "fork_staging"
|
||||
SEO_REPORTS_DIR = DATA_DIR / "seo_reports"
|
||||
DEEPSEARCH_DIR = DATA_DIR / "deepsearch"
|
||||
DEEPSEARCH_CHROMA_DIR = DEEPSEARCH_DIR / "chroma"
|
||||
KEYS_DIR = DATA_DIR / "keys"
|
||||
BOT_DIR = DATA_DIR / "bot"
|
||||
LOCKS_DIR = DATA_DIR / "locks"
|
||||
@@ -52,6 +54,7 @@ INTERNAL_BASE_URL = environ.get(
|
||||
"DEVPLACE_INTERNAL_BASE_URL", f"http://localhost:{PORT}"
|
||||
).rstrip("/")
|
||||
INTERNAL_GATEWAY_URL = f"{INTERNAL_BASE_URL}/openai/v1/chat/completions"
|
||||
INTERNAL_EMBED_URL = f"{INTERNAL_BASE_URL}/openai/v1/embeddings"
|
||||
INTERNAL_MODEL = "molodetz"
|
||||
INTERNAL_EMBED_MODEL = "molodetz~embed"
|
||||
|
||||
@@ -86,6 +89,8 @@ DATA_PATHS: dict[str, Path] = {
|
||||
"zip_staging": ZIP_STAGING_DIR,
|
||||
"fork_staging": FORK_STAGING_DIR,
|
||||
"seo_reports": SEO_REPORTS_DIR,
|
||||
"deepsearch": DEEPSEARCH_DIR,
|
||||
"deepsearch_chroma": DEEPSEARCH_CHROMA_DIR,
|
||||
"keys": KEYS_DIR,
|
||||
"bot": BOT_DIR,
|
||||
"locks": LOCKS_DIR,
|
||||
|
||||
@@ -138,6 +138,8 @@ SOFT_DELETE_TABLES = [
|
||||
"bug_tickets",
|
||||
"bug_comment_authors",
|
||||
"notification_preferences",
|
||||
"deepsearch_sessions",
|
||||
"deepsearch_messages",
|
||||
]
|
||||
|
||||
|
||||
@@ -456,6 +458,63 @@ def init_db():
|
||||
["user_uid", "notification_type"],
|
||||
)
|
||||
|
||||
deepsearch_sessions = get_table("deepsearch_sessions")
|
||||
for column, example in (
|
||||
("uid", ""),
|
||||
("owner_kind", ""),
|
||||
("owner_id", ""),
|
||||
("query", ""),
|
||||
("status", ""),
|
||||
("depth", 0),
|
||||
("max_pages", 0),
|
||||
("score", 0),
|
||||
("confidence", 0.0),
|
||||
("source_diversity", 0.0),
|
||||
("page_count", 0),
|
||||
("chunk_count", 0),
|
||||
("collection", ""),
|
||||
("summary", ""),
|
||||
("created_at", ""),
|
||||
("completed_at", ""),
|
||||
("deleted_at", ""),
|
||||
("deleted_by", ""),
|
||||
):
|
||||
if not deepsearch_sessions.has_column(column):
|
||||
deepsearch_sessions.create_column_by_example(column, example)
|
||||
_index(db, "deepsearch_sessions", "idx_deepsearch_sessions_owner", ["owner_kind", "owner_id"])
|
||||
_index(db, "deepsearch_sessions", "idx_deepsearch_sessions_status", ["status"])
|
||||
_index(db, "deepsearch_sessions", "idx_deepsearch_sessions_created", ["created_at"])
|
||||
|
||||
deepsearch_messages = get_table("deepsearch_messages")
|
||||
for column, example in (
|
||||
("uid", ""),
|
||||
("session_uid", ""),
|
||||
("role", ""),
|
||||
("content", ""),
|
||||
("citations", ""),
|
||||
("created_at", ""),
|
||||
("deleted_at", ""),
|
||||
("deleted_by", ""),
|
||||
):
|
||||
if not deepsearch_messages.has_column(column):
|
||||
deepsearch_messages.create_column_by_example(column, example)
|
||||
_index(db, "deepsearch_messages", "idx_deepsearch_messages_session", ["session_uid"])
|
||||
|
||||
deepsearch_url_cache = get_table("deepsearch_url_cache")
|
||||
for column, example in (
|
||||
("uid", ""),
|
||||
("url_hash", ""),
|
||||
("url", ""),
|
||||
("title", ""),
|
||||
("content_hash", ""),
|
||||
("status", 0),
|
||||
("byte_size", 0),
|
||||
("fetched_at", ""),
|
||||
):
|
||||
if not deepsearch_url_cache.has_column(column):
|
||||
deepsearch_url_cache.create_column_by_example(column, example)
|
||||
_index(db, "deepsearch_url_cache", "idx_deepsearch_url_cache_hash", ["url_hash"])
|
||||
|
||||
for table in SOFT_DELETE_TABLES:
|
||||
ensure_soft_delete_columns(table)
|
||||
|
||||
@@ -2313,3 +2372,119 @@ def get_featured_news(limit=5):
|
||||
}
|
||||
)
|
||||
return articles
|
||||
|
||||
|
||||
def _ds_now() -> str:
|
||||
return datetime.now(timezone.utc).isoformat()
|
||||
|
||||
|
||||
def create_deepsearch_session(
|
||||
uid: str,
|
||||
owner_kind: str,
|
||||
owner_id: str,
|
||||
query: str,
|
||||
depth: int,
|
||||
max_pages: int,
|
||||
collection: str,
|
||||
) -> None:
|
||||
get_table("deepsearch_sessions").insert(
|
||||
{
|
||||
"uid": uid,
|
||||
"owner_kind": owner_kind,
|
||||
"owner_id": owner_id,
|
||||
"query": query,
|
||||
"status": "pending",
|
||||
"depth": depth,
|
||||
"max_pages": max_pages,
|
||||
"score": 0,
|
||||
"confidence": 0.0,
|
||||
"source_diversity": 0.0,
|
||||
"page_count": 0,
|
||||
"chunk_count": 0,
|
||||
"collection": collection,
|
||||
"summary": "",
|
||||
"created_at": _ds_now(),
|
||||
"completed_at": "",
|
||||
"deleted_at": None,
|
||||
"deleted_by": None,
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def update_deepsearch_session(uid: str, fields: dict) -> None:
|
||||
if "deepsearch_sessions" not in db.tables:
|
||||
return
|
||||
payload = dict(fields)
|
||||
payload["uid"] = uid
|
||||
get_table("deepsearch_sessions").update(payload, ["uid"])
|
||||
|
||||
|
||||
def get_deepsearch_session(uid: str) -> dict | None:
|
||||
if "deepsearch_sessions" not in db.tables:
|
||||
return None
|
||||
return get_table("deepsearch_sessions").find_one(uid=uid, deleted_at=None)
|
||||
|
||||
|
||||
def add_deepsearch_message(
|
||||
uid: str, session_uid: str, role: str, content: str, citations: str = ""
|
||||
) -> None:
|
||||
get_table("deepsearch_messages").insert(
|
||||
{
|
||||
"uid": uid,
|
||||
"session_uid": session_uid,
|
||||
"role": role,
|
||||
"content": content,
|
||||
"citations": citations,
|
||||
"created_at": _ds_now(),
|
||||
"deleted_at": None,
|
||||
"deleted_by": None,
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def get_deepsearch_messages(session_uid: str, limit: int = 50) -> list[dict]:
|
||||
if "deepsearch_messages" not in db.tables:
|
||||
return []
|
||||
return list(
|
||||
get_table("deepsearch_messages").find(
|
||||
session_uid=session_uid,
|
||||
deleted_at=None,
|
||||
order_by=["created_at"],
|
||||
_limit=limit,
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def get_cached_deepsearch_url(url_hash: str) -> dict | None:
|
||||
if "deepsearch_url_cache" not in db.tables:
|
||||
return None
|
||||
return get_table("deepsearch_url_cache").find_one(url_hash=url_hash)
|
||||
|
||||
|
||||
def upsert_deepsearch_url_cache(
|
||||
url_hash: str,
|
||||
url: str,
|
||||
title: str,
|
||||
content_hash: str,
|
||||
status: int,
|
||||
byte_size: int,
|
||||
) -> None:
|
||||
table = get_table("deepsearch_url_cache")
|
||||
existing = table.find_one(url_hash=url_hash)
|
||||
row = {
|
||||
"url_hash": url_hash,
|
||||
"url": url,
|
||||
"title": title,
|
||||
"content_hash": content_hash,
|
||||
"status": status,
|
||||
"byte_size": byte_size,
|
||||
"fetched_at": _ds_now(),
|
||||
}
|
||||
if existing:
|
||||
row["uid"] = existing["uid"]
|
||||
table.update(row, ["uid"])
|
||||
else:
|
||||
from devplacepy.utils import generate_uid
|
||||
|
||||
row["uid"] = generate_uid()
|
||||
table.insert(row)
|
||||
|
||||
@@ -3342,6 +3342,86 @@ status and report.
|
||||
field("index", "path", "integer", True, "0", "Zero-based index of the audited page."),
|
||||
],
|
||||
),
|
||||
endpoint(
|
||||
id="tools-deepsearch-run",
|
||||
method="POST",
|
||||
path="/tools/deepsearch/run",
|
||||
title="Queue a DeepSearch research job",
|
||||
summary="Start a multi-agent deep web research job. Returns the job uid plus status and websocket URLs.",
|
||||
auth="public",
|
||||
encoding="form",
|
||||
params=[
|
||||
field("query", "form", "string", True, "history of the transistor", "The research question to investigate."),
|
||||
field("depth", "form", "integer", False, "2", "Research depth (1-4)."),
|
||||
field("max_pages", "form", "integer", False, "12", "Maximum sources to crawl (1-30)."),
|
||||
],
|
||||
sample_response={
|
||||
"uid": "DEEPSEARCH_JOB_UID",
|
||||
"status_url": "/tools/deepsearch/DEEPSEARCH_JOB_UID",
|
||||
"ws_url": "/tools/deepsearch/DEEPSEARCH_JOB_UID/ws",
|
||||
},
|
||||
),
|
||||
endpoint(
|
||||
id="tools-deepsearch-status",
|
||||
method="GET",
|
||||
path="/tools/deepsearch/{uid}",
|
||||
title="DeepSearch status",
|
||||
summary="Poll a DeepSearch job. Once done, score, confidence and session_url are populated.",
|
||||
auth="public",
|
||||
params=[
|
||||
field("uid", "path", "string", True, "DEEPSEARCH_JOB_UID", "DeepSearch job uid returned when the run was queued."),
|
||||
],
|
||||
sample_response={
|
||||
"uid": "DEEPSEARCH_JOB_UID",
|
||||
"kind": "deepsearch",
|
||||
"status": "done",
|
||||
"query": "history of the transistor",
|
||||
"depth": 2,
|
||||
"max_pages": 12,
|
||||
"ws_url": "/tools/deepsearch/DEEPSEARCH_JOB_UID/ws",
|
||||
"chat_ws_url": "/tools/deepsearch/DEEPSEARCH_JOB_UID/chat",
|
||||
"session_url": "/tools/deepsearch/DEEPSEARCH_JOB_UID/session",
|
||||
"score": 78,
|
||||
"confidence": 0.72,
|
||||
"source_diversity": 0.64,
|
||||
"page_count": 11,
|
||||
"chunk_count": 240,
|
||||
"error": None,
|
||||
"created_at": "2026-06-14T10:00:00+00:00",
|
||||
"completed_at": "2026-06-14T10:01:40+00:00",
|
||||
},
|
||||
),
|
||||
endpoint(
|
||||
id="tools-deepsearch-session",
|
||||
method="GET",
|
||||
path="/tools/deepsearch/{uid}/session",
|
||||
title="DeepSearch report",
|
||||
summary="Full cited research report: summary, findings, gaps, sources and metrics. Negotiates HTML or JSON.",
|
||||
auth="public",
|
||||
params=[
|
||||
field("uid", "path", "string", True, "DEEPSEARCH_JOB_UID", "DeepSearch job uid of a finished run."),
|
||||
],
|
||||
sample_response={
|
||||
"uid": "DEEPSEARCH_JOB_UID",
|
||||
"status": "done",
|
||||
"query": "history of the transistor",
|
||||
"score": 78,
|
||||
"confidence": 0.72,
|
||||
"source_diversity": 0.64,
|
||||
"page_count": 11,
|
||||
"chunk_count": 240,
|
||||
"summary": "The transistor was invented at Bell Labs in 1947...",
|
||||
"findings": [
|
||||
{"title": "Invention", "detail": "...", "confidence": 0.8, "citations": [1]}
|
||||
],
|
||||
"gaps": ["Limited coverage of later MOSFET developments."],
|
||||
"sources": [{"url": "https://example.com", "title": "Example", "source": "httpx"}],
|
||||
"chat_ws_url": "/tools/deepsearch/DEEPSEARCH_JOB_UID/chat",
|
||||
"export_md_url": "/tools/deepsearch/DEEPSEARCH_JOB_UID/export.md",
|
||||
"export_json_url": "/tools/deepsearch/DEEPSEARCH_JOB_UID/export.json",
|
||||
"export_pdf_url": "/tools/deepsearch/DEEPSEARCH_JOB_UID/export.pdf",
|
||||
},
|
||||
),
|
||||
],
|
||||
},
|
||||
{
|
||||
|
||||
@@ -81,6 +81,7 @@ from devplacepy.services.jobs.zip_service import ZipService
|
||||
from devplacepy.services.jobs.fork_service import ForkService
|
||||
from devplacepy.services.jobs.bug_create_service import BugCreateService
|
||||
from devplacepy.services.jobs.seo.service import SeoService
|
||||
from devplacepy.services.jobs.deepsearch.service import DeepsearchService
|
||||
from devplacepy.services.gitea.service import BugTrackerService
|
||||
from devplacepy.services.containers.service import ContainerService
|
||||
from devplacepy.services.audit import AuditService
|
||||
@@ -438,6 +439,7 @@ async def startup():
|
||||
service_manager.register(ZipService())
|
||||
service_manager.register(ForkService())
|
||||
service_manager.register(SeoService())
|
||||
service_manager.register(DeepsearchService())
|
||||
service_manager.register(BugCreateService())
|
||||
service_manager.register(BugTrackerService())
|
||||
service_manager.register(ContainerService())
|
||||
|
||||
@@ -384,6 +384,51 @@ class SeoRunForm(BaseModel):
|
||||
return text
|
||||
|
||||
|
||||
DEEPSEARCH_MIN_DEPTH = 1
|
||||
DEEPSEARCH_MAX_DEPTH = 4
|
||||
DEEPSEARCH_DEFAULT_DEPTH = 2
|
||||
DEEPSEARCH_MIN_PAGES = 1
|
||||
DEEPSEARCH_MAX_PAGES = 30
|
||||
DEEPSEARCH_DEFAULT_PAGES = 12
|
||||
DEEPSEARCH_MIN_QUERY = 3
|
||||
DEEPSEARCH_MAX_QUERY = 500
|
||||
DEEPSEARCH_MAX_MESSAGE = 2000
|
||||
|
||||
|
||||
class DeepsearchRunForm(BaseModel):
|
||||
query: str = Field(min_length=DEEPSEARCH_MIN_QUERY, max_length=DEEPSEARCH_MAX_QUERY)
|
||||
depth: int = Field(
|
||||
default=DEEPSEARCH_DEFAULT_DEPTH,
|
||||
ge=DEEPSEARCH_MIN_DEPTH,
|
||||
le=DEEPSEARCH_MAX_DEPTH,
|
||||
)
|
||||
max_pages: int = Field(
|
||||
default=DEEPSEARCH_DEFAULT_PAGES,
|
||||
ge=DEEPSEARCH_MIN_PAGES,
|
||||
le=DEEPSEARCH_MAX_PAGES,
|
||||
)
|
||||
|
||||
@field_validator("query")
|
||||
@classmethod
|
||||
def query_present(cls, value):
|
||||
text = value.strip()
|
||||
if not text:
|
||||
raise ValueError("A research question is required")
|
||||
return text
|
||||
|
||||
|
||||
class DeepsearchChatForm(BaseModel):
|
||||
message: str = Field(min_length=1, max_length=DEEPSEARCH_MAX_MESSAGE)
|
||||
|
||||
@field_validator("message")
|
||||
@classmethod
|
||||
def message_present(cls, value):
|
||||
text = value.strip()
|
||||
if not text:
|
||||
raise ValueError("A message is required")
|
||||
return text
|
||||
|
||||
|
||||
class AdminRoleForm(BaseModel):
|
||||
role: Literal["member", "admin"]
|
||||
|
||||
|
||||
@@ -78,6 +78,12 @@ DOCS_PAGES = [
|
||||
"kind": "prose",
|
||||
"section": SECTION_TOOLS,
|
||||
},
|
||||
{
|
||||
"slug": "tools-deepsearch",
|
||||
"title": "DeepSearch",
|
||||
"kind": "prose",
|
||||
"section": SECTION_TOOLS,
|
||||
},
|
||||
# Claude Code - the native subagent, command, and workflow setup under .claude/
|
||||
{
|
||||
"slug": "claude",
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
# retoor <retoor@molodetz.nl>
|
||||
|
||||
from . import index, seo
|
||||
from . import deepsearch, index, seo
|
||||
|
||||
router = index.router
|
||||
router.include_router(seo.router, prefix="/seo")
|
||||
router.include_router(deepsearch.router, prefix="/deepsearch")
|
||||
|
||||
@@ -0,0 +1,15 @@
|
||||
# retoor <retoor@molodetz.nl>
|
||||
|
||||
from fastapi import Request
|
||||
|
||||
from devplacepy.utils import get_current_user
|
||||
|
||||
|
||||
def owner_for(request: Request) -> tuple[str, str]:
|
||||
user = get_current_user(request)
|
||||
if user:
|
||||
return "user", user["uid"]
|
||||
ip = request.headers.get("X-Real-IP") or (
|
||||
request.client.host if request.client else "anon"
|
||||
)
|
||||
return "guest", ip
|
||||
@@ -0,0 +1,430 @@
|
||||
# retoor <retoor@molodetz.nl>
|
||||
|
||||
import json
|
||||
import logging
|
||||
from typing import Annotated
|
||||
|
||||
from fastapi import APIRouter, Form, Request, WebSocket, WebSocketDisconnect
|
||||
from fastapi.responses import HTMLResponse, JSONResponse, Response
|
||||
|
||||
from devplacepy import database
|
||||
from devplacepy.config import DEEPSEARCH_DIR
|
||||
from devplacepy.models import DeepsearchChatForm, DeepsearchRunForm
|
||||
from devplacepy.responses import respond
|
||||
from devplacepy.schemas import DeepsearchJobOut, DeepsearchSessionOut
|
||||
from devplacepy.seo import base_seo_context, site_url, web_application_schema, website_schema
|
||||
from devplacepy.services.deepsearch.chat import DeepsearchChat
|
||||
from devplacepy.services.deepsearch.export import to_json, to_markdown, to_pdf
|
||||
from devplacepy.services.jobs import queue
|
||||
from devplacepy.services.jobs.deepsearch.progress import hub
|
||||
from devplacepy.services.jobs.deepsearch.service import DeepsearchService
|
||||
from devplacepy.services.manager import service_manager
|
||||
from devplacepy.templating import templates
|
||||
from devplacepy.utils import generate_uid, get_current_user, is_admin, not_found
|
||||
|
||||
from ._shared import owner_for
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
router = APIRouter()
|
||||
|
||||
ACTIVE_STATES = (queue.PENDING, queue.RUNNING)
|
||||
TERMINAL_STATES = (queue.DONE, queue.FAILED)
|
||||
CONTROL_STATES = ("running", "paused", "cancelled")
|
||||
TOUCH_EXTEND_SECONDS = 3600
|
||||
_service = DeepsearchService()
|
||||
|
||||
|
||||
def _collection_for(uid: str) -> str:
|
||||
return _service.collection_name(uid)
|
||||
|
||||
|
||||
def _control_path(uid: str):
|
||||
return DEEPSEARCH_DIR / uid / "control.json"
|
||||
|
||||
|
||||
def _set_control(uid: str, state: str) -> None:
|
||||
path = _control_path(uid)
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
path.write_text(json.dumps({"state": state}), encoding="utf-8")
|
||||
|
||||
|
||||
def _owns(request: Request, session: dict) -> bool:
|
||||
owner_kind, owner_id = owner_for(request)
|
||||
return (
|
||||
session.get("owner_kind") == owner_kind
|
||||
and session.get("owner_id") == owner_id
|
||||
)
|
||||
|
||||
|
||||
def _job_payload(job: dict) -> dict:
|
||||
result = job.get("result", {})
|
||||
payload = job.get("payload", {})
|
||||
done = job.get("status") == queue.DONE
|
||||
uid = job.get("uid", "")
|
||||
return {
|
||||
"uid": uid,
|
||||
"kind": job.get("kind", ""),
|
||||
"status": job.get("status", ""),
|
||||
"query": payload.get("query"),
|
||||
"depth": int(payload.get("depth") or 0),
|
||||
"max_pages": int(payload.get("max_pages") or 0),
|
||||
"ws_url": f"/tools/deepsearch/{uid}/ws",
|
||||
"chat_ws_url": f"/tools/deepsearch/{uid}/chat" if done else None,
|
||||
"session_url": f"/tools/deepsearch/{uid}/session" if done else None,
|
||||
"score": result.get("score") if done else None,
|
||||
"confidence": result.get("confidence") if done else None,
|
||||
"source_diversity": result.get("source_diversity") if done else None,
|
||||
"page_count": int(result.get("page_count") or 0),
|
||||
"chunk_count": int(result.get("chunk_count") or 0),
|
||||
"error": job.get("error") or None,
|
||||
"created_at": job.get("created_at"),
|
||||
"completed_at": job.get("completed_at") or None,
|
||||
}
|
||||
|
||||
|
||||
@router.get("", response_class=HTMLResponse)
|
||||
async def deepsearch_page(request: Request):
|
||||
user = get_current_user(request)
|
||||
base = site_url(request)
|
||||
description = (
|
||||
"Run a multi-agent deep web research job: it plans queries, crawls and indexes "
|
||||
"sources, then synthesises a cited report with confidence scoring and gap "
|
||||
"analysis, plus a grounded chat over the results."
|
||||
)
|
||||
seo_ctx = base_seo_context(
|
||||
request,
|
||||
title="DeepSearch",
|
||||
description=description,
|
||||
breadcrumbs=[
|
||||
{"name": "Home", "url": "/feed"},
|
||||
{"name": "Tools", "url": "/tools"},
|
||||
{"name": "DeepSearch", "url": "/tools/deepsearch"},
|
||||
],
|
||||
schemas=[
|
||||
website_schema(base),
|
||||
web_application_schema("DeepSearch", description, "/tools/deepsearch", base),
|
||||
],
|
||||
)
|
||||
return templates.TemplateResponse(
|
||||
request,
|
||||
"tools/deepsearch.html",
|
||||
{**seo_ctx, "request": request, "user": user},
|
||||
)
|
||||
|
||||
|
||||
@router.post("/run")
|
||||
async def deepsearch_run(request: Request, data: Annotated[DeepsearchRunForm, Form()]):
|
||||
owner_kind, owner_id = owner_for(request)
|
||||
from devplacepy.services.audit import record as audit
|
||||
|
||||
active = [
|
||||
job
|
||||
for job in queue.list_jobs(kind="deepsearch", owner=(owner_kind, owner_id))
|
||||
if job.get("status") in ACTIVE_STATES
|
||||
]
|
||||
if active:
|
||||
audit.record(
|
||||
request,
|
||||
"deepsearch.run.request",
|
||||
result="denied",
|
||||
summary=f"DeepSearch denied for {data.query}: already running",
|
||||
metadata={"query": data.query, "reason": "active_job", "uid": active[0]["uid"]},
|
||||
links=[audit.job(active[0]["uid"])],
|
||||
)
|
||||
return JSONResponse(
|
||||
{
|
||||
"error": {
|
||||
"status": 429,
|
||||
"message": "You already have a DeepSearch running. Wait for it to finish.",
|
||||
"uid": active[0]["uid"],
|
||||
}
|
||||
},
|
||||
status_code=429,
|
||||
)
|
||||
user = get_current_user(request)
|
||||
api_key = (
|
||||
user.get("api_key") if user else ""
|
||||
) or database.internal_gateway_key()
|
||||
uid = generate_uid()
|
||||
payload = {
|
||||
"query": data.query,
|
||||
"depth": data.depth,
|
||||
"max_pages": data.max_pages,
|
||||
"api_key": api_key,
|
||||
"collection": _collection_for(uid),
|
||||
}
|
||||
database.create_deepsearch_session(
|
||||
uid, owner_kind, owner_id, data.query, data.depth, data.max_pages, _collection_for(uid)
|
||||
)
|
||||
_enqueue(uid, payload, owner_kind, owner_id, data.query)
|
||||
audit.record(
|
||||
request,
|
||||
"deepsearch.run.request",
|
||||
summary=f"requested DeepSearch for {data.query}",
|
||||
metadata={"query": data.query, "depth": data.depth, "max_pages": data.max_pages},
|
||||
links=[audit.job(uid)],
|
||||
)
|
||||
return JSONResponse(
|
||||
{
|
||||
"uid": uid,
|
||||
"status_url": f"/tools/deepsearch/{uid}",
|
||||
"ws_url": f"/tools/deepsearch/{uid}/ws",
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def _enqueue(uid: str, payload: dict, owner_kind: str, owner_id: str, query: str) -> None:
|
||||
from datetime import datetime, timezone
|
||||
|
||||
now = datetime.now(timezone.utc).isoformat()
|
||||
database.get_table("jobs").insert(
|
||||
{
|
||||
"uid": uid,
|
||||
"kind": "deepsearch",
|
||||
"status": queue.PENDING,
|
||||
"owner_kind": owner_kind,
|
||||
"owner_id": owner_id,
|
||||
"preferred_name": f"DeepSearch: {query}"[:64],
|
||||
"payload": json.dumps(payload),
|
||||
"result": "",
|
||||
"error": "",
|
||||
"retry_count": 0,
|
||||
"created_at": now,
|
||||
"started_at": "",
|
||||
"completed_at": "",
|
||||
"updated_at": now,
|
||||
"duration_ms": 0,
|
||||
"last_accessed_at": "",
|
||||
"expires_at": "",
|
||||
"bytes_in": 0,
|
||||
"bytes_out": 0,
|
||||
"item_count": 0,
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
@router.get("/{uid}")
|
||||
async def deepsearch_status(request: Request, uid: str):
|
||||
job = queue.get_job(uid)
|
||||
if not job or job.get("kind") != "deepsearch":
|
||||
raise not_found("Research not found")
|
||||
return JSONResponse(
|
||||
DeepsearchJobOut.model_validate(_job_payload(job)).model_dump(mode="json")
|
||||
)
|
||||
|
||||
|
||||
def _report_for(job: dict) -> dict:
|
||||
if job.get("status") != queue.DONE:
|
||||
return {}
|
||||
return job.get("result", {}).get("report", {})
|
||||
|
||||
|
||||
def _session_context(request: Request, uid: str, job: dict, session: dict) -> dict:
|
||||
report = _report_for(job)
|
||||
user = get_current_user(request)
|
||||
viewer_is_admin = is_admin(user)
|
||||
done = job.get("status") == queue.DONE
|
||||
return {
|
||||
"uid": uid,
|
||||
"status": job.get("status", ""),
|
||||
"query": report.get("query") or session.get("query"),
|
||||
"depth": int(session.get("depth") or 0),
|
||||
"max_pages": int(session.get("max_pages") or 0),
|
||||
"score": report.get("score"),
|
||||
"confidence": report.get("confidence"),
|
||||
"source_diversity": report.get("source_diversity"),
|
||||
"page_count": report.get("page_count", 0),
|
||||
"chunk_count": report.get("chunk_count", 0),
|
||||
"summary": report.get("summary", ""),
|
||||
"sources": report.get("sources", []),
|
||||
"findings": report.get("findings", []),
|
||||
"gaps": report.get("gaps", []),
|
||||
"timeline": report.get("timeline", []),
|
||||
"chat_ws_url": f"/tools/deepsearch/{uid}/chat" if done else None,
|
||||
"export_md_url": f"/tools/deepsearch/{uid}/export.md" if done else None,
|
||||
"export_json_url": f"/tools/deepsearch/{uid}/export.json" if done else None,
|
||||
"export_pdf_url": f"/tools/deepsearch/{uid}/export.pdf" if done else None,
|
||||
"viewer_is_admin": viewer_is_admin,
|
||||
"viewer_owns": _owns(request, session),
|
||||
"created_at": session.get("created_at"),
|
||||
"completed_at": session.get("completed_at") or None,
|
||||
"request": request,
|
||||
"user": user,
|
||||
"meta_robots": "noindex,nofollow",
|
||||
}
|
||||
|
||||
|
||||
@router.get("/{uid}/session")
|
||||
async def deepsearch_session(request: Request, uid: str):
|
||||
job = queue.get_job(uid)
|
||||
session = database.get_deepsearch_session(uid)
|
||||
if not job or job.get("kind") != "deepsearch" or not session:
|
||||
raise not_found("Research not found")
|
||||
queue.touch_job(uid, TOUCH_EXTEND_SECONDS)
|
||||
context = _session_context(request, uid, job, session)
|
||||
return respond(request, "tools/deepsearch_session.html", context, model=DeepsearchSessionOut)
|
||||
|
||||
|
||||
@router.post("/{uid}/pause")
|
||||
async def deepsearch_pause(request: Request, uid: str):
|
||||
return _control(request, uid, "paused")
|
||||
|
||||
|
||||
@router.post("/{uid}/resume")
|
||||
async def deepsearch_resume(request: Request, uid: str):
|
||||
return _control(request, uid, "running")
|
||||
|
||||
|
||||
@router.post("/{uid}/cancel")
|
||||
async def deepsearch_cancel(request: Request, uid: str):
|
||||
return _control(request, uid, "cancelled")
|
||||
|
||||
|
||||
def _control(request: Request, uid: str, state: str):
|
||||
session = database.get_deepsearch_session(uid)
|
||||
job = queue.get_job(uid)
|
||||
if not job or job.get("kind") != "deepsearch" or not session:
|
||||
raise not_found("Research not found")
|
||||
if not _owns(request, session):
|
||||
return JSONResponse(
|
||||
{"error": {"status": 403, "message": "Not your research session."}},
|
||||
status_code=403,
|
||||
)
|
||||
_set_control(uid, state)
|
||||
return JSONResponse({"ok": True, "state": state})
|
||||
|
||||
|
||||
def _export_report(uid: str) -> dict | None:
|
||||
job = queue.get_job(uid)
|
||||
if not job or job.get("kind") != "deepsearch" or job.get("status") != queue.DONE:
|
||||
return None
|
||||
queue.touch_job(uid, TOUCH_EXTEND_SECONDS)
|
||||
return job.get("result", {}).get("report", {})
|
||||
|
||||
|
||||
@router.get("/{uid}/export.md")
|
||||
async def deepsearch_export_md(request: Request, uid: str):
|
||||
report = _export_report(uid)
|
||||
if report is None:
|
||||
raise not_found("Report not ready")
|
||||
return Response(content=to_markdown(report), media_type="text/markdown")
|
||||
|
||||
|
||||
@router.get("/{uid}/export.json")
|
||||
async def deepsearch_export_json(request: Request, uid: str):
|
||||
report = _export_report(uid)
|
||||
if report is None:
|
||||
raise not_found("Report not ready")
|
||||
return Response(content=to_json(report), media_type="application/json")
|
||||
|
||||
|
||||
@router.get("/{uid}/export.pdf")
|
||||
async def deepsearch_export_pdf(request: Request, uid: str):
|
||||
report = _export_report(uid)
|
||||
if report is None:
|
||||
raise not_found("Report not ready")
|
||||
return Response(content=to_pdf(report), media_type="application/pdf")
|
||||
|
||||
|
||||
@router.websocket("/{uid}/ws")
|
||||
async def deepsearch_ws(websocket: WebSocket, uid: str):
|
||||
await websocket.accept()
|
||||
if not service_manager.owns_lock():
|
||||
await websocket.close(code=4013)
|
||||
return
|
||||
job = queue.get_job(uid)
|
||||
if not job or job.get("kind") != "deepsearch":
|
||||
await websocket.close(code=1008)
|
||||
return
|
||||
listener = hub.subscribe(uid)
|
||||
try:
|
||||
for frame in hub.snapshot(uid):
|
||||
await websocket.send_json(frame)
|
||||
current = queue.get_job(uid)
|
||||
if current and current.get("status") in TERMINAL_STATES:
|
||||
done = current.get("status") == queue.DONE
|
||||
await websocket.send_json(
|
||||
{
|
||||
"type": "done" if done else "failed",
|
||||
"status": current.get("status"),
|
||||
"session_url": f"/tools/deepsearch/{uid}/session",
|
||||
"error": current.get("error") or None,
|
||||
}
|
||||
)
|
||||
return
|
||||
while True:
|
||||
frame = await listener.get()
|
||||
await websocket.send_json(frame)
|
||||
if frame.get("type") in ("done", "failed"):
|
||||
break
|
||||
except WebSocketDisconnect:
|
||||
pass
|
||||
except Exception:
|
||||
logger.exception("DeepSearch websocket loop failed for %s", uid)
|
||||
finally:
|
||||
hub.unsubscribe(uid, listener)
|
||||
|
||||
|
||||
@router.websocket("/{uid}/chat")
|
||||
async def deepsearch_chat_ws(websocket: WebSocket, uid: str):
|
||||
await websocket.accept()
|
||||
if not service_manager.owns_lock():
|
||||
await websocket.close(code=4013)
|
||||
return
|
||||
job = queue.get_job(uid)
|
||||
session = database.get_deepsearch_session(uid)
|
||||
if not job or job.get("kind") != "deepsearch" or job.get("status") != queue.DONE or not session:
|
||||
await websocket.close(code=1008)
|
||||
return
|
||||
user = get_current_user(websocket)
|
||||
api_key = (
|
||||
user.get("api_key") if user else ""
|
||||
) or database.internal_gateway_key()
|
||||
chat = DeepsearchChat(_collection_for(uid), api_key)
|
||||
history: list[dict] = []
|
||||
for row in database.get_deepsearch_messages(uid):
|
||||
history.append({"role": row.get("role"), "content": row.get("content")})
|
||||
await websocket.send_json({"type": "ready", "history": history})
|
||||
from devplacepy.services.audit import record as audit
|
||||
|
||||
try:
|
||||
while True:
|
||||
raw = await websocket.receive_text()
|
||||
try:
|
||||
frame = json.loads(raw)
|
||||
except (ValueError, TypeError):
|
||||
continue
|
||||
if frame.get("type") != "input":
|
||||
continue
|
||||
try:
|
||||
form = DeepsearchChatForm.model_validate({"message": frame.get("text", "")})
|
||||
except Exception:
|
||||
await websocket.send_json({"type": "error", "text": "Empty message."})
|
||||
continue
|
||||
question = form.message
|
||||
await websocket.send_json({"type": "status", "text": "Searching the research collection"})
|
||||
database.add_deepsearch_message(generate_uid(), uid, "user", question)
|
||||
history.append({"role": "user", "content": question})
|
||||
answer = await chat.answer(question, history)
|
||||
database.add_deepsearch_message(
|
||||
generate_uid(),
|
||||
uid,
|
||||
"assistant",
|
||||
answer.text,
|
||||
json.dumps(answer.citations),
|
||||
)
|
||||
history.append({"role": "assistant", "content": answer.text})
|
||||
await websocket.send_json(
|
||||
{"type": "reply", "text": answer.text, "citations": answer.citations}
|
||||
)
|
||||
audit.record(
|
||||
websocket,
|
||||
"deepsearch.chat",
|
||||
summary=f"DeepSearch chat on session {uid[:8]}",
|
||||
metadata={"session": uid},
|
||||
links=[audit.job(uid)],
|
||||
)
|
||||
except WebSocketDisconnect:
|
||||
pass
|
||||
except Exception:
|
||||
logger.exception("DeepSearch chat loop failed for %s", uid)
|
||||
@@ -23,6 +23,17 @@ TOOLS = [
|
||||
"structured-data, performance, accessibility and AI-readiness checks."
|
||||
),
|
||||
},
|
||||
{
|
||||
"slug": "deepsearch",
|
||||
"name": "DeepSearch",
|
||||
"icon": "🧠",
|
||||
"url": "/tools/deepsearch",
|
||||
"description": (
|
||||
"Run a multi-agent deep web research job that crawls and indexes sources, "
|
||||
"synthesises a cited report with confidence and gap analysis, then lets you "
|
||||
"chat over the gathered evidence."
|
||||
),
|
||||
},
|
||||
]
|
||||
|
||||
|
||||
|
||||
@@ -17,6 +17,7 @@ from devplacepy.services.jobs.seo.progress import hub
|
||||
from devplacepy.services.manager import service_manager
|
||||
from devplacepy.templating import templates
|
||||
from devplacepy.utils import get_current_user, not_found
|
||||
from ._shared import owner_for as _owner
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
router = APIRouter()
|
||||
@@ -25,16 +26,6 @@ ACTIVE_STATES = (queue.PENDING, queue.RUNNING)
|
||||
TERMINAL_STATES = (queue.DONE, queue.FAILED)
|
||||
|
||||
|
||||
def _owner(request: Request) -> tuple[str, str]:
|
||||
user = get_current_user(request)
|
||||
if user:
|
||||
return "user", user["uid"]
|
||||
ip = request.headers.get("X-Real-IP") or (
|
||||
request.client.host if request.client else "anon"
|
||||
)
|
||||
return "guest", ip
|
||||
|
||||
|
||||
def _status_payload(job: dict) -> dict:
|
||||
result = job.get("result", {})
|
||||
done = job.get("status") == queue.DONE
|
||||
|
||||
@@ -127,6 +127,52 @@ class SeoReportOut(_Out):
|
||||
generated_at: Optional[str] = None
|
||||
|
||||
|
||||
class DeepsearchJobOut(_Out):
|
||||
uid: str = ""
|
||||
kind: str = ""
|
||||
status: str = ""
|
||||
query: Optional[str] = None
|
||||
depth: int = 0
|
||||
max_pages: int = 0
|
||||
ws_url: Optional[str] = None
|
||||
chat_ws_url: Optional[str] = None
|
||||
session_url: Optional[str] = None
|
||||
score: Optional[int] = None
|
||||
confidence: Optional[float] = None
|
||||
source_diversity: Optional[float] = None
|
||||
page_count: int = 0
|
||||
chunk_count: int = 0
|
||||
error: Optional[str] = None
|
||||
created_at: Optional[str] = None
|
||||
completed_at: Optional[str] = None
|
||||
|
||||
|
||||
class DeepsearchSessionOut(_Out):
|
||||
uid: str = ""
|
||||
status: str = ""
|
||||
query: Optional[str] = None
|
||||
depth: int = 0
|
||||
max_pages: int = 0
|
||||
score: Optional[int] = None
|
||||
confidence: Optional[float] = None
|
||||
source_diversity: Optional[float] = None
|
||||
page_count: int = 0
|
||||
chunk_count: int = 0
|
||||
summary: Optional[str] = None
|
||||
sources: list = []
|
||||
findings: list = []
|
||||
gaps: list = []
|
||||
timeline: list = []
|
||||
chat_ws_url: Optional[str] = None
|
||||
export_md_url: Optional[str] = None
|
||||
export_json_url: Optional[str] = None
|
||||
export_pdf_url: Optional[str] = None
|
||||
viewer_is_admin: bool = False
|
||||
viewer_owns: bool = False
|
||||
created_at: Optional[str] = None
|
||||
completed_at: Optional[str] = None
|
||||
|
||||
|
||||
# ---------- shared resource projections ----------
|
||||
|
||||
|
||||
|
||||
@@ -349,6 +349,7 @@ def _build_sitemap(base_url):
|
||||
urlset.append(url_element(f"{base_url}/bugs", changefreq="daily", priority="0.6"))
|
||||
urlset.append(url_element(f"{base_url}/tools", changefreq="monthly", priority="0.5"))
|
||||
urlset.append(url_element(f"{base_url}/tools/seo", changefreq="monthly", priority="0.5"))
|
||||
urlset.append(url_element(f"{base_url}/tools/deepsearch", changefreq="monthly", priority="0.5"))
|
||||
|
||||
if "posts" in db.tables:
|
||||
posts = _collect(
|
||||
|
||||
@@ -27,6 +27,7 @@ CATEGORY_BY_PREFIX: dict[str, str] = {
|
||||
"container": "container",
|
||||
"proxy": "ingress",
|
||||
"seo": "tools",
|
||||
"deepsearch": "tools",
|
||||
"ai": "ai",
|
||||
"devii": "devii",
|
||||
"cli": "cli",
|
||||
|
||||
@@ -0,0 +1 @@
|
||||
# retoor <retoor@molodetz.nl>
|
||||
@@ -0,0 +1,107 @@
|
||||
# retoor <retoor@molodetz.nl>
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from dataclasses import dataclass, field
|
||||
|
||||
from .embeddings import embed_texts, local_embed
|
||||
from .llm import complete_chat
|
||||
from .store import Chunk, VectorStore
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
CHAT_TOP_K = 8
|
||||
MAX_CONTEXT_CHARS = 9000
|
||||
CHAT_MAX_TOKENS = 900
|
||||
|
||||
SYSTEM_PROMPT = (
|
||||
"You are the DeepSearch research assistant. Answer the user's question using ONLY "
|
||||
"the numbered SOURCES below, which were gathered during a web research session. "
|
||||
"Never use outside knowledge or guess. If the sources do not contain the answer, "
|
||||
"say so plainly. Cite every claim inline with the bracket marker of the source it "
|
||||
"comes from, like [1] or [2]. Keep the answer focused and well structured in "
|
||||
"markdown."
|
||||
)
|
||||
|
||||
|
||||
@dataclass
|
||||
class ChatAnswer:
|
||||
text: str
|
||||
citations: list[dict] = field(default_factory=list)
|
||||
|
||||
|
||||
def _build_context(chunks: list[Chunk]) -> tuple[str, list[dict]]:
|
||||
blocks: list[str] = []
|
||||
citations: list[dict] = []
|
||||
used = 0
|
||||
for index, chunk in enumerate(chunks, start=1):
|
||||
snippet = chunk.text.strip()
|
||||
if not snippet:
|
||||
continue
|
||||
header = f"[{index}] {chunk.title or chunk.url} ({chunk.url})"
|
||||
block = f"{header}\n{snippet}"
|
||||
if used + len(block) > MAX_CONTEXT_CHARS and blocks:
|
||||
break
|
||||
used += len(block)
|
||||
blocks.append(block)
|
||||
citations.append(
|
||||
{
|
||||
"index": index,
|
||||
"url": chunk.url,
|
||||
"title": chunk.title or chunk.url,
|
||||
"score": round(chunk.score, 4),
|
||||
}
|
||||
)
|
||||
return "\n\n".join(blocks), citations
|
||||
|
||||
|
||||
class DeepsearchChat:
|
||||
def __init__(self, collection_name: str, api_key: str) -> None:
|
||||
self.store = VectorStore(collection_name)
|
||||
self.api_key = api_key
|
||||
|
||||
async def _embed_query(self, question: str) -> list[float]:
|
||||
result = await embed_texts([question], self.api_key)
|
||||
if not result.vectors:
|
||||
result = local_embed([question])
|
||||
return result.vectors[0]
|
||||
|
||||
async def retrieve(self, question: str) -> list[Chunk]:
|
||||
query_vector = await self._embed_query(question)
|
||||
return self.store.hybrid_search(question, query_vector, top_k=CHAT_TOP_K)
|
||||
|
||||
async def answer(self, question: str, history: list[dict] | None = None) -> ChatAnswer:
|
||||
chunks = await self.retrieve(question)
|
||||
if not chunks:
|
||||
return ChatAnswer(
|
||||
text=(
|
||||
"The research session did not capture anything relevant to that "
|
||||
"question. Try rephrasing or running a deeper search."
|
||||
),
|
||||
citations=[],
|
||||
)
|
||||
context, citations = _build_context(chunks)
|
||||
messages = [{"role": "system", "content": SYSTEM_PROMPT}]
|
||||
for turn in (history or [])[-6:]:
|
||||
role = turn.get("role")
|
||||
content = turn.get("content")
|
||||
if role in ("user", "assistant") and content:
|
||||
messages.append({"role": role, "content": content})
|
||||
messages.append(
|
||||
{
|
||||
"role": "user",
|
||||
"content": f"SOURCES:\n{context}\n\nQUESTION: {question}",
|
||||
}
|
||||
)
|
||||
try:
|
||||
text = await complete_chat(
|
||||
messages, self.api_key, max_tokens=CHAT_MAX_TOKENS
|
||||
)
|
||||
except Exception as exc:
|
||||
logger.warning("deepsearch chat synthesis failed: %s", exc)
|
||||
text = (
|
||||
"I could not reach the language model to synthesise an answer, but the "
|
||||
"most relevant sources are listed below."
|
||||
)
|
||||
return ChatAnswer(text=text, citations=citations)
|
||||
@@ -0,0 +1,71 @@
|
||||
# retoor <retoor@molodetz.nl>
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import logging
|
||||
import math
|
||||
import re
|
||||
from dataclasses import dataclass
|
||||
|
||||
import httpx
|
||||
|
||||
from devplacepy.config import INTERNAL_EMBED_MODEL, INTERNAL_EMBED_URL
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
EMBED_TIMEOUT_SECONDS = 60.0
|
||||
LOCAL_EMBED_DIMS = 256
|
||||
TOKEN_PATTERN = re.compile(r"[a-z0-9]+")
|
||||
|
||||
|
||||
@dataclass
|
||||
class EmbedResult:
|
||||
vectors: list[list[float]]
|
||||
backend: str
|
||||
|
||||
|
||||
def _local_vector(text: str) -> list[float]:
|
||||
bucket = [0.0] * LOCAL_EMBED_DIMS
|
||||
tokens = TOKEN_PATTERN.findall((text or "").lower())
|
||||
if not tokens:
|
||||
return bucket
|
||||
for token in tokens:
|
||||
digest = hashlib.sha1(token.encode("utf-8")).digest()
|
||||
index = int.from_bytes(digest[:4], "big") % LOCAL_EMBED_DIMS
|
||||
sign = 1.0 if digest[4] % 2 == 0 else -1.0
|
||||
bucket[index] += sign
|
||||
norm = math.sqrt(sum(value * value for value in bucket))
|
||||
if norm == 0.0:
|
||||
return bucket
|
||||
return [value / norm for value in bucket]
|
||||
|
||||
|
||||
def local_embed(texts: list[str]) -> EmbedResult:
|
||||
return EmbedResult(vectors=[_local_vector(text) for text in texts], backend="local")
|
||||
|
||||
|
||||
async def embed_texts(
|
||||
texts: list[str], api_key: str, *, gateway_url: str = INTERNAL_EMBED_URL
|
||||
) -> EmbedResult:
|
||||
if not texts:
|
||||
return EmbedResult(vectors=[], backend="empty")
|
||||
headers = {
|
||||
"Authorization": f"Bearer {api_key}",
|
||||
"Content-Type": "application/json",
|
||||
}
|
||||
payload = {"model": INTERNAL_EMBED_MODEL, "input": texts}
|
||||
try:
|
||||
async with httpx.AsyncClient(timeout=EMBED_TIMEOUT_SECONDS) as client:
|
||||
response = await client.post(gateway_url, json=payload, headers=headers)
|
||||
if response.status_code >= 400:
|
||||
raise RuntimeError(f"embed gateway returned {response.status_code}")
|
||||
data = response.json()
|
||||
rows = data.get("data") or []
|
||||
vectors = [row.get("embedding") or [] for row in rows]
|
||||
if len(vectors) != len(texts) or any(not vector for vector in vectors):
|
||||
raise RuntimeError("embed gateway returned an incomplete response")
|
||||
return EmbedResult(vectors=vectors, backend="gateway")
|
||||
except Exception as exc:
|
||||
logger.warning("deepsearch embedding gateway failed, using local: %s", exc)
|
||||
return local_embed(texts)
|
||||
@@ -0,0 +1,132 @@
|
||||
# retoor <retoor@molodetz.nl>
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import html
|
||||
import json
|
||||
import logging
|
||||
from datetime import datetime, timezone
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _findings(report: dict) -> list[dict]:
|
||||
return report.get("findings") or []
|
||||
|
||||
|
||||
def _sources(report: dict) -> list[dict]:
|
||||
return report.get("sources") or []
|
||||
|
||||
|
||||
def _gaps(report: dict) -> list[str]:
|
||||
return report.get("gaps") or []
|
||||
|
||||
|
||||
def to_markdown(report: dict) -> str:
|
||||
query = report.get("query", "")
|
||||
lines: list[str] = [f"# DeepSearch report: {query}", ""]
|
||||
lines.append(
|
||||
f"- Score: {report.get('score', 0)} "
|
||||
f"Confidence: {report.get('confidence', 0)} "
|
||||
f"Source diversity: {report.get('source_diversity', 0)}"
|
||||
)
|
||||
lines.append(
|
||||
f"- Pages crawled: {report.get('page_count', 0)} "
|
||||
f"Chunks indexed: {report.get('chunk_count', 0)}"
|
||||
)
|
||||
generated = report.get("generated_at") or datetime.now(timezone.utc).isoformat()
|
||||
lines.append(f"- Generated: {generated}")
|
||||
lines.append("")
|
||||
summary = report.get("summary", "")
|
||||
if summary:
|
||||
lines.extend(["## Summary", "", summary, ""])
|
||||
findings = _findings(report)
|
||||
if findings:
|
||||
lines.append("## Findings")
|
||||
lines.append("")
|
||||
for finding in findings:
|
||||
title = finding.get("title", "")
|
||||
detail = finding.get("detail", "")
|
||||
confidence = finding.get("confidence", 0)
|
||||
lines.append(f"### {title}")
|
||||
lines.append("")
|
||||
lines.append(detail)
|
||||
citations = finding.get("citations") or []
|
||||
if citations:
|
||||
lines.append("")
|
||||
lines.append("Sources: " + ", ".join(str(c) for c in citations))
|
||||
lines.append(f"\nConfidence: {confidence}")
|
||||
lines.append("")
|
||||
gaps = _gaps(report)
|
||||
if gaps:
|
||||
lines.append("## Open gaps")
|
||||
lines.append("")
|
||||
for gap in gaps:
|
||||
lines.append(f"- {gap}")
|
||||
lines.append("")
|
||||
sources = _sources(report)
|
||||
if sources:
|
||||
lines.append("## Sources")
|
||||
lines.append("")
|
||||
for index, source in enumerate(sources, start=1):
|
||||
title = source.get("title") or source.get("url", "")
|
||||
url = source.get("url", "")
|
||||
lines.append(f"{index}. [{title}]({url})")
|
||||
lines.append("")
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
def to_json(report: dict) -> str:
|
||||
return json.dumps(report, ensure_ascii=False, indent=2)
|
||||
|
||||
|
||||
def _html_document(report: dict) -> str:
|
||||
query = html.escape(report.get("query", ""))
|
||||
parts: list[str] = [
|
||||
"<html><head><meta charset='utf-8'><style>",
|
||||
"body{font-family:Arial,Helvetica,sans-serif;color:#1b2330;margin:40px;}",
|
||||
"h1{font-size:22px;}h2{font-size:17px;margin-top:24px;}h3{font-size:14px;}",
|
||||
".meta{color:#566;font-size:12px;}a{color:#2d6cdf;}",
|
||||
"</style></head><body>",
|
||||
f"<h1>DeepSearch report: {query}</h1>",
|
||||
"<p class='meta'>"
|
||||
f"Score {report.get('score', 0)} | Confidence {report.get('confidence', 0)} | "
|
||||
f"Diversity {report.get('source_diversity', 0)} | "
|
||||
f"Pages {report.get('page_count', 0)} | Chunks {report.get('chunk_count', 0)}"
|
||||
"</p>",
|
||||
]
|
||||
summary = report.get("summary", "")
|
||||
if summary:
|
||||
parts.append("<h2>Summary</h2>")
|
||||
parts.append(f"<p>{html.escape(summary)}</p>")
|
||||
findings = _findings(report)
|
||||
if findings:
|
||||
parts.append("<h2>Findings</h2>")
|
||||
for finding in findings:
|
||||
parts.append(f"<h3>{html.escape(finding.get('title', ''))}</h3>")
|
||||
parts.append(f"<p>{html.escape(finding.get('detail', ''))}</p>")
|
||||
parts.append(
|
||||
f"<p class='meta'>Confidence {finding.get('confidence', 0)}</p>"
|
||||
)
|
||||
gaps = _gaps(report)
|
||||
if gaps:
|
||||
parts.append("<h2>Open gaps</h2><ul>")
|
||||
for gap in gaps:
|
||||
parts.append(f"<li>{html.escape(gap)}</li>")
|
||||
parts.append("</ul>")
|
||||
sources = _sources(report)
|
||||
if sources:
|
||||
parts.append("<h2>Sources</h2><ol>")
|
||||
for source in sources:
|
||||
url = html.escape(source.get("url", ""))
|
||||
title = html.escape(source.get("title") or source.get("url", ""))
|
||||
parts.append(f"<li><a href='{url}'>{title}</a></li>")
|
||||
parts.append("</ol>")
|
||||
parts.append("</body></html>")
|
||||
return "".join(parts)
|
||||
|
||||
|
||||
def to_pdf(report: dict) -> bytes:
|
||||
from weasyprint import HTML
|
||||
|
||||
return HTML(string=_html_document(report)).write_pdf()
|
||||
@@ -0,0 +1,44 @@
|
||||
# retoor <retoor@molodetz.nl>
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
|
||||
import httpx
|
||||
|
||||
from devplacepy.config import INTERNAL_GATEWAY_URL, INTERNAL_MODEL
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
CHAT_TIMEOUT_SECONDS = 120.0
|
||||
DEFAULT_MAX_TOKENS = 1200
|
||||
|
||||
|
||||
async def complete_chat(
|
||||
messages: list[dict],
|
||||
api_key: str,
|
||||
*,
|
||||
gateway_url: str = INTERNAL_GATEWAY_URL,
|
||||
model: str = INTERNAL_MODEL,
|
||||
max_tokens: int = DEFAULT_MAX_TOKENS,
|
||||
temperature: float = 0.2,
|
||||
) -> str:
|
||||
payload = {
|
||||
"model": model,
|
||||
"messages": messages,
|
||||
"max_tokens": max_tokens,
|
||||
"temperature": temperature,
|
||||
}
|
||||
headers = {
|
||||
"Authorization": f"Bearer {api_key}",
|
||||
"Content-Type": "application/json",
|
||||
}
|
||||
async with httpx.AsyncClient(timeout=CHAT_TIMEOUT_SECONDS) as client:
|
||||
response = await client.post(gateway_url, json=payload, headers=headers)
|
||||
if response.status_code >= 400:
|
||||
raise RuntimeError(f"chat gateway returned {response.status_code}")
|
||||
data = response.json()
|
||||
choices = data.get("choices") or []
|
||||
if not choices:
|
||||
raise RuntimeError("chat gateway returned no choices")
|
||||
return (choices[0].get("message", {}).get("content") or "").strip()
|
||||
@@ -0,0 +1,209 @@
|
||||
# retoor <retoor@molodetz.nl>
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import math
|
||||
import re
|
||||
from collections import Counter
|
||||
from dataclasses import dataclass, field
|
||||
|
||||
from devplacepy.config import DEEPSEARCH_CHROMA_DIR
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
TOKEN_PATTERN = re.compile(r"[a-z0-9]+")
|
||||
BM25_K1 = 1.5
|
||||
BM25_B = 0.75
|
||||
HYBRID_VECTOR_WEIGHT = 0.6
|
||||
HYBRID_KEYWORD_WEIGHT = 0.4
|
||||
DEFAULT_TOP_K = 8
|
||||
CANDIDATE_MULTIPLIER = 4
|
||||
|
||||
|
||||
@dataclass
|
||||
class Chunk:
|
||||
uid: str
|
||||
text: str
|
||||
url: str
|
||||
title: str
|
||||
depth: int = 0
|
||||
source: str = ""
|
||||
position: int = 0
|
||||
score: float = 0.0
|
||||
metadata: dict = field(default_factory=dict)
|
||||
|
||||
|
||||
def _tokenize(text: str) -> list[str]:
|
||||
return TOKEN_PATTERN.findall((text or "").lower())
|
||||
|
||||
|
||||
class VectorStore:
|
||||
def __init__(self, collection_name: str) -> None:
|
||||
self.collection_name = collection_name
|
||||
self._client = None
|
||||
self._collection = None
|
||||
|
||||
def _ensure(self):
|
||||
if self._collection is not None:
|
||||
return self._collection
|
||||
import chromadb
|
||||
|
||||
DEEPSEARCH_CHROMA_DIR.mkdir(parents=True, exist_ok=True)
|
||||
self._client = chromadb.PersistentClient(path=str(DEEPSEARCH_CHROMA_DIR))
|
||||
self._collection = self._client.get_or_create_collection(
|
||||
name=self.collection_name, metadata={"hnsw:space": "cosine"}
|
||||
)
|
||||
return self._collection
|
||||
|
||||
def add(self, chunks: list[Chunk], vectors: list[list[float]]) -> None:
|
||||
if not chunks:
|
||||
return
|
||||
collection = self._ensure()
|
||||
collection.add(
|
||||
ids=[chunk.uid for chunk in chunks],
|
||||
embeddings=vectors,
|
||||
documents=[chunk.text for chunk in chunks],
|
||||
metadatas=[
|
||||
{
|
||||
"url": chunk.url,
|
||||
"title": chunk.title,
|
||||
"depth": chunk.depth,
|
||||
"source": chunk.source,
|
||||
"position": chunk.position,
|
||||
}
|
||||
for chunk in chunks
|
||||
],
|
||||
)
|
||||
|
||||
def all_chunks(self) -> list[Chunk]:
|
||||
collection = self._ensure()
|
||||
data = collection.get(include=["documents", "metadatas"])
|
||||
chunks: list[Chunk] = []
|
||||
ids = data.get("ids") or []
|
||||
documents = data.get("documents") or []
|
||||
metadatas = data.get("metadatas") or []
|
||||
for index, uid in enumerate(ids):
|
||||
meta = metadatas[index] if index < len(metadatas) else {}
|
||||
chunks.append(
|
||||
Chunk(
|
||||
uid=uid,
|
||||
text=documents[index] if index < len(documents) else "",
|
||||
url=str(meta.get("url", "")),
|
||||
title=str(meta.get("title", "")),
|
||||
depth=int(meta.get("depth", 0) or 0),
|
||||
source=str(meta.get("source", "")),
|
||||
position=int(meta.get("position", 0) or 0),
|
||||
metadata=dict(meta),
|
||||
)
|
||||
)
|
||||
return chunks
|
||||
|
||||
def count(self) -> int:
|
||||
try:
|
||||
return self._ensure().count()
|
||||
except Exception:
|
||||
return 0
|
||||
|
||||
def vector_search(
|
||||
self, query_vector: list[float], top_k: int, where: dict | None = None
|
||||
) -> list[Chunk]:
|
||||
collection = self._ensure()
|
||||
result = collection.query(
|
||||
query_embeddings=[query_vector],
|
||||
n_results=top_k,
|
||||
where=where or None,
|
||||
include=["documents", "metadatas", "distances"],
|
||||
)
|
||||
ids = (result.get("ids") or [[]])[0]
|
||||
documents = (result.get("documents") or [[]])[0]
|
||||
metadatas = (result.get("metadatas") or [[]])[0]
|
||||
distances = (result.get("distances") or [[]])[0]
|
||||
chunks: list[Chunk] = []
|
||||
for index, uid in enumerate(ids):
|
||||
meta = metadatas[index] if index < len(metadatas) else {}
|
||||
distance = distances[index] if index < len(distances) else 1.0
|
||||
chunks.append(
|
||||
Chunk(
|
||||
uid=uid,
|
||||
text=documents[index] if index < len(documents) else "",
|
||||
url=str(meta.get("url", "")),
|
||||
title=str(meta.get("title", "")),
|
||||
depth=int(meta.get("depth", 0) or 0),
|
||||
source=str(meta.get("source", "")),
|
||||
position=int(meta.get("position", 0) or 0),
|
||||
score=1.0 - float(distance),
|
||||
metadata=dict(meta),
|
||||
)
|
||||
)
|
||||
return chunks
|
||||
|
||||
def keyword_scores(self, query: str, chunks: list[Chunk]) -> dict[str, float]:
|
||||
terms = _tokenize(query)
|
||||
if not terms or not chunks:
|
||||
return {}
|
||||
docs = [_tokenize(chunk.text) for chunk in chunks]
|
||||
lengths = [len(doc) for doc in docs]
|
||||
avg_len = (sum(lengths) / len(lengths)) if lengths else 0.0
|
||||
doc_freq: Counter = Counter()
|
||||
for doc in docs:
|
||||
for term in set(doc):
|
||||
if term in terms:
|
||||
doc_freq[term] += 1
|
||||
total_docs = len(docs)
|
||||
scores: dict[str, float] = {}
|
||||
for index, chunk in enumerate(chunks):
|
||||
counts = Counter(docs[index])
|
||||
length = lengths[index] or 1
|
||||
score = 0.0
|
||||
for term in terms:
|
||||
freq = counts.get(term, 0)
|
||||
if freq == 0:
|
||||
continue
|
||||
idf = math.log(
|
||||
1 + (total_docs - doc_freq[term] + 0.5) / (doc_freq[term] + 0.5)
|
||||
)
|
||||
denom = freq + BM25_K1 * (
|
||||
1 - BM25_B + BM25_B * (length / (avg_len or 1))
|
||||
)
|
||||
score += idf * (freq * (BM25_K1 + 1)) / (denom or 1)
|
||||
scores[chunk.uid] = score
|
||||
return scores
|
||||
|
||||
def hybrid_search(
|
||||
self,
|
||||
query: str,
|
||||
query_vector: list[float],
|
||||
top_k: int = DEFAULT_TOP_K,
|
||||
where: dict | None = None,
|
||||
) -> list[Chunk]:
|
||||
candidates = self.vector_search(
|
||||
query_vector, top_k * CANDIDATE_MULTIPLIER, where
|
||||
)
|
||||
if not candidates:
|
||||
return []
|
||||
keyword = self.keyword_scores(query, candidates)
|
||||
vec_max = max((chunk.score for chunk in candidates), default=0.0) or 1.0
|
||||
kw_max = max(keyword.values(), default=0.0) or 1.0
|
||||
for chunk in candidates:
|
||||
vec_norm = max(0.0, chunk.score) / vec_max
|
||||
kw_norm = keyword.get(chunk.uid, 0.0) / kw_max
|
||||
chunk.score = (
|
||||
HYBRID_VECTOR_WEIGHT * vec_norm + HYBRID_KEYWORD_WEIGHT * kw_norm
|
||||
)
|
||||
candidates.sort(key=lambda chunk: chunk.score, reverse=True)
|
||||
return candidates[:top_k]
|
||||
|
||||
def drop(self) -> None:
|
||||
try:
|
||||
import chromadb
|
||||
|
||||
DEEPSEARCH_CHROMA_DIR.mkdir(parents=True, exist_ok=True)
|
||||
client = self._client or chromadb.PersistentClient(
|
||||
path=str(DEEPSEARCH_CHROMA_DIR)
|
||||
)
|
||||
client.delete_collection(self.collection_name)
|
||||
except Exception as exc:
|
||||
logger.info("deepsearch collection drop skipped for %s: %s", self.collection_name, exc)
|
||||
finally:
|
||||
self._collection = None
|
||||
@@ -595,6 +595,48 @@ ACTIONS: tuple[Action, ...] = (
|
||||
params=(path("uid", "SEO job uid returned by seo_diagnostics."),),
|
||||
requires_auth=False,
|
||||
),
|
||||
Action(
|
||||
name="deepsearch",
|
||||
method="POST",
|
||||
path="/tools/deepsearch/run",
|
||||
summary="Start a deep multi-agent web research job",
|
||||
description=(
|
||||
"Queues a background DeepSearch job and returns {uid, status_url}. Poll the "
|
||||
"status_url with deepsearch_status until status is 'done', then share the "
|
||||
"score, confidence and session_url. depth (1-4) and max_pages (1-30) control "
|
||||
"how widely it crawls."
|
||||
),
|
||||
params=(
|
||||
body("query", "The research question to investigate.", required=True),
|
||||
body("depth", "Research depth 1-4 (default 2)."),
|
||||
body("max_pages", "Maximum sources to crawl 1-30 (default 12)."),
|
||||
),
|
||||
requires_auth=False,
|
||||
),
|
||||
Action(
|
||||
name="deepsearch_status",
|
||||
method="GET",
|
||||
path="/tools/deepsearch/{uid}",
|
||||
summary="Check a DeepSearch job and obtain its metrics once finished",
|
||||
description=(
|
||||
"Returns the research status. When status is 'done', score, confidence, "
|
||||
"source_diversity and session_url are populated; otherwise poll again shortly."
|
||||
),
|
||||
params=(path("uid", "DeepSearch job uid returned by deepsearch."),),
|
||||
requires_auth=False,
|
||||
),
|
||||
Action(
|
||||
name="deepsearch_session",
|
||||
method="GET",
|
||||
path="/tools/deepsearch/{uid}/session",
|
||||
summary="Read a finished DeepSearch report",
|
||||
description=(
|
||||
"Returns the full cited report for a finished DeepSearch: summary, findings, "
|
||||
"gaps, sources and metrics."
|
||||
),
|
||||
params=(path("uid", "DeepSearch job uid returned by deepsearch."),),
|
||||
requires_auth=False,
|
||||
),
|
||||
Action(
|
||||
name="search_users",
|
||||
method="GET",
|
||||
|
||||
@@ -0,0 +1 @@
|
||||
# retoor <retoor@molodetz.nl>
|
||||
@@ -0,0 +1,49 @@
|
||||
# retoor <retoor@molodetz.nl>
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
|
||||
CHUNK_TARGET_CHARS = 1200
|
||||
CHUNK_OVERLAP_CHARS = 150
|
||||
MIN_CHUNK_CHARS = 80
|
||||
PARAGRAPH_SPLIT = re.compile(r"\n\s*\n")
|
||||
WHITESPACE = re.compile(r"[ \t]+")
|
||||
|
||||
|
||||
def normalize_text(text: str) -> str:
|
||||
cleaned = (text or "").replace("\r", "\n")
|
||||
cleaned = WHITESPACE.sub(" ", cleaned)
|
||||
lines = [line.strip() for line in cleaned.split("\n")]
|
||||
return "\n".join(line for line in lines if line)
|
||||
|
||||
|
||||
def chunk_text(text: str) -> list[str]:
|
||||
normalized = normalize_text(text)
|
||||
if not normalized:
|
||||
return []
|
||||
paragraphs = [p.strip() for p in PARAGRAPH_SPLIT.split(normalized) if p.strip()]
|
||||
if not paragraphs:
|
||||
paragraphs = [normalized]
|
||||
chunks: list[str] = []
|
||||
buffer = ""
|
||||
for paragraph in paragraphs:
|
||||
if len(paragraph) > CHUNK_TARGET_CHARS:
|
||||
if buffer:
|
||||
chunks.append(buffer)
|
||||
buffer = ""
|
||||
for start in range(0, len(paragraph), CHUNK_TARGET_CHARS - CHUNK_OVERLAP_CHARS):
|
||||
piece = paragraph[start : start + CHUNK_TARGET_CHARS]
|
||||
if len(piece) >= MIN_CHUNK_CHARS:
|
||||
chunks.append(piece)
|
||||
continue
|
||||
if len(buffer) + len(paragraph) + 1 > CHUNK_TARGET_CHARS and buffer:
|
||||
chunks.append(buffer)
|
||||
buffer = paragraph
|
||||
else:
|
||||
buffer = f"{buffer}\n{paragraph}" if buffer else paragraph
|
||||
if buffer and len(buffer) >= MIN_CHUNK_CHARS:
|
||||
chunks.append(buffer)
|
||||
elif buffer and not chunks:
|
||||
chunks.append(buffer)
|
||||
return chunks
|
||||
@@ -0,0 +1,214 @@
|
||||
# retoor <retoor@molodetz.nl>
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import logging
|
||||
import re
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Awaitable, Callable
|
||||
|
||||
import httpx
|
||||
|
||||
from devplacepy.net_guard import BlockedAddressError, guard_public_url, guarded_async_client
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
RSEARCH_URL = "https://rsearch.app.molodetz.nl"
|
||||
RSEARCH_TIMEOUT_SECONDS = 45.0
|
||||
FETCH_TIMEOUT_SECONDS = 20.0
|
||||
MAX_FETCH_BYTES = 2_500_000
|
||||
RESULTS_PER_QUERY = 8
|
||||
USER_AGENT = (
|
||||
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) "
|
||||
"Chrome/131.0.0.0 Safari/537.36 DevPlaceDeepSearchBot/1.0"
|
||||
)
|
||||
SCRIPT_STYLE = re.compile(r"<(script|style)[^>]*>.*?</\1>", re.DOTALL | re.IGNORECASE)
|
||||
TAG = re.compile(r"<[^>]+>")
|
||||
TITLE = re.compile(r"<title[^>]*>(.*?)</title>", re.DOTALL | re.IGNORECASE)
|
||||
SPACE = re.compile(r"\s+")
|
||||
MIN_PAGE_CHARS = 200
|
||||
|
||||
|
||||
@dataclass
|
||||
class CrawledPage:
|
||||
url: str
|
||||
title: str
|
||||
text: str
|
||||
source: str
|
||||
status: int
|
||||
depth: int = 0
|
||||
from_cache: bool = False
|
||||
|
||||
|
||||
@dataclass
|
||||
class CrawlOutcome:
|
||||
pages: list[CrawledPage] = field(default_factory=list)
|
||||
seen_hashes: set[str] = field(default_factory=set)
|
||||
|
||||
|
||||
def url_hash(url: str) -> str:
|
||||
return hashlib.sha256(url.strip().lower().encode("utf-8")).hexdigest()
|
||||
|
||||
|
||||
def content_hash(text: str) -> str:
|
||||
return hashlib.sha256(text.strip().encode("utf-8")).hexdigest()
|
||||
|
||||
|
||||
def _strip_html(raw: str) -> tuple[str, str]:
|
||||
title_match = TITLE.search(raw)
|
||||
title = SPACE.sub(" ", TAG.sub("", title_match.group(1))).strip() if title_match else ""
|
||||
body = SCRIPT_STYLE.sub(" ", raw)
|
||||
body = TAG.sub(" ", body)
|
||||
body = SPACE.sub(" ", body).strip()
|
||||
return title, body
|
||||
|
||||
|
||||
async def search_queries(queries: list[str]) -> list[dict]:
|
||||
results: list[dict] = []
|
||||
seen: set[str] = set()
|
||||
headers = {"User-Agent": USER_AGENT, "Accept": "application/json"}
|
||||
timeout = httpx.Timeout(RSEARCH_TIMEOUT_SECONDS, connect=30.0)
|
||||
async with httpx.AsyncClient(
|
||||
base_url=RSEARCH_URL, headers=headers, follow_redirects=True, timeout=timeout
|
||||
) as client:
|
||||
for query in queries:
|
||||
try:
|
||||
response = await client.get(
|
||||
"/search",
|
||||
params={"query": query, "count": RESULTS_PER_QUERY, "content": "false"},
|
||||
)
|
||||
if response.status_code >= 400:
|
||||
continue
|
||||
data = response.json()
|
||||
except (httpx.HTTPError, ValueError) as exc:
|
||||
logger.warning("deepsearch rsearch failed for %r: %s", query, exc)
|
||||
continue
|
||||
for item in data.get("results") or []:
|
||||
url = (item.get("url") or "").strip()
|
||||
if not url or url in seen:
|
||||
continue
|
||||
seen.add(url)
|
||||
results.append(
|
||||
{
|
||||
"url": url,
|
||||
"title": item.get("title") or "",
|
||||
"description": item.get("description") or "",
|
||||
}
|
||||
)
|
||||
return results
|
||||
|
||||
|
||||
async def _render_with_playwright(url: str) -> tuple[str, str, int]:
|
||||
from playwright.async_api import async_playwright
|
||||
|
||||
async with async_playwright() as pw:
|
||||
browser = await pw.chromium.launch(
|
||||
headless=True, args=["--no-sandbox", "--disable-dev-shm-usage"]
|
||||
)
|
||||
try:
|
||||
context = await browser.new_context(user_agent=USER_AGENT)
|
||||
page = await context.new_page()
|
||||
response = await page.goto(url, wait_until="load", timeout=30000)
|
||||
status = response.status if response else 0
|
||||
for hop in [response.url] if response else []:
|
||||
await guard_public_url(hop)
|
||||
content = (await page.content())[:MAX_FETCH_BYTES]
|
||||
await context.close()
|
||||
title, text = _strip_html(content)
|
||||
return title, text, status
|
||||
finally:
|
||||
await browser.close()
|
||||
|
||||
|
||||
async def fetch_page(url: str, depth: int) -> CrawledPage | None:
|
||||
try:
|
||||
await guard_public_url(url)
|
||||
except BlockedAddressError:
|
||||
return None
|
||||
title = ""
|
||||
text = ""
|
||||
status = 0
|
||||
source = "httpx"
|
||||
try:
|
||||
async with guarded_async_client(
|
||||
follow_redirects=True,
|
||||
timeout=FETCH_TIMEOUT_SECONDS,
|
||||
headers={"User-Agent": USER_AGENT},
|
||||
) as client:
|
||||
response = await client.get(url)
|
||||
await guard_public_url(str(response.url))
|
||||
status = response.status_code
|
||||
raw = response.text[:MAX_FETCH_BYTES]
|
||||
title, text = _strip_html(raw)
|
||||
except (httpx.HTTPError, BlockedAddressError) as exc:
|
||||
logger.info("deepsearch httpx fetch failed for %s: %s", url, exc)
|
||||
if len(text) < MIN_PAGE_CHARS:
|
||||
try:
|
||||
r_title, r_text, r_status = await _render_with_playwright(url)
|
||||
if len(r_text) > len(text):
|
||||
title, text, status, source = (
|
||||
r_title or title,
|
||||
r_text,
|
||||
r_status or status,
|
||||
"playwright",
|
||||
)
|
||||
except Exception as exc:
|
||||
logger.info("deepsearch render failed for %s: %s", url, exc)
|
||||
if len(text) < MIN_PAGE_CHARS:
|
||||
return None
|
||||
return CrawledPage(
|
||||
url=url, title=title or url, text=text, source=source, status=status, depth=depth
|
||||
)
|
||||
|
||||
|
||||
async def crawl(
|
||||
candidates: list[dict],
|
||||
max_pages: int,
|
||||
emit: Callable[[dict], None],
|
||||
is_cached: Callable[[str], bool],
|
||||
should_stop: Callable[[], Awaitable[bool]],
|
||||
) -> CrawlOutcome:
|
||||
outcome = CrawlOutcome()
|
||||
fetched = 0
|
||||
total = min(len(candidates), max_pages)
|
||||
for index, candidate in enumerate(candidates):
|
||||
if fetched >= max_pages:
|
||||
break
|
||||
if await should_stop():
|
||||
emit({"type": "stage", "stage": "cancelled", "message": "Crawl cancelled"})
|
||||
break
|
||||
url = candidate["url"]
|
||||
emit(
|
||||
{
|
||||
"type": "progress",
|
||||
"done": fetched,
|
||||
"total": total,
|
||||
"url": url,
|
||||
"message": f"Fetching {url}",
|
||||
}
|
||||
)
|
||||
if is_cached(url):
|
||||
emit({"type": "page_cached", "url": url})
|
||||
page = await fetch_page(url, depth=0)
|
||||
if page is None:
|
||||
emit({"type": "page_skipped", "url": url})
|
||||
continue
|
||||
digest = content_hash(page.text)
|
||||
if digest in outcome.seen_hashes:
|
||||
emit({"type": "page_duplicate", "url": url})
|
||||
continue
|
||||
outcome.seen_hashes.add(digest)
|
||||
outcome.pages.append(page)
|
||||
fetched += 1
|
||||
emit(
|
||||
{
|
||||
"type": "page_loaded",
|
||||
"url": page.url,
|
||||
"title": page.title,
|
||||
"source": page.source,
|
||||
"done": fetched,
|
||||
"total": total,
|
||||
}
|
||||
)
|
||||
return outcome
|
||||
@@ -0,0 +1,86 @@
|
||||
# retoor <retoor@molodetz.nl>
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
import re
|
||||
|
||||
import httpx
|
||||
|
||||
from devplacepy.config import INTERNAL_GATEWAY_URL, INTERNAL_MODEL
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
ENHANCE_TIMEOUT_SECONDS = 90.0
|
||||
ENHANCE_MAX_TOKENS = 400
|
||||
MAX_SUBQUERIES = 6
|
||||
|
||||
PLANNER_PROMPT = (
|
||||
"You plan web research. Given a research question, produce a JSON object with one "
|
||||
"key 'queries': an array of 3 to 6 concise, diverse web search queries that together "
|
||||
"cover the question from multiple angles. Return ONLY the JSON object, no prose."
|
||||
)
|
||||
|
||||
|
||||
def _fallback(query: str) -> list[str]:
|
||||
base = query.strip()
|
||||
variants = [
|
||||
base,
|
||||
f"{base} overview",
|
||||
f"{base} latest",
|
||||
f"{base} explained",
|
||||
]
|
||||
seen: list[str] = []
|
||||
for variant in variants:
|
||||
if variant and variant not in seen:
|
||||
seen.append(variant)
|
||||
return seen[:MAX_SUBQUERIES]
|
||||
|
||||
|
||||
def _parse(text: str) -> list[str]:
|
||||
match = re.search(r"\{.*\}", text, re.DOTALL)
|
||||
if not match:
|
||||
return []
|
||||
try:
|
||||
payload = json.loads(match.group())
|
||||
except (ValueError, TypeError):
|
||||
return []
|
||||
queries = payload.get("queries")
|
||||
if not isinstance(queries, list):
|
||||
return []
|
||||
cleaned = [str(item).strip() for item in queries if str(item).strip()]
|
||||
return cleaned[:MAX_SUBQUERIES]
|
||||
|
||||
|
||||
async def plan_queries(query: str, api_key: str) -> list[str]:
|
||||
payload = {
|
||||
"model": INTERNAL_MODEL,
|
||||
"messages": [
|
||||
{"role": "system", "content": PLANNER_PROMPT},
|
||||
{"role": "user", "content": query},
|
||||
],
|
||||
"max_tokens": ENHANCE_MAX_TOKENS,
|
||||
"temperature": 0.3,
|
||||
}
|
||||
headers = {
|
||||
"Authorization": f"Bearer {api_key}",
|
||||
"Content-Type": "application/json",
|
||||
}
|
||||
try:
|
||||
async with httpx.AsyncClient(timeout=ENHANCE_TIMEOUT_SECONDS) as client:
|
||||
response = await client.post(
|
||||
INTERNAL_GATEWAY_URL, json=payload, headers=headers
|
||||
)
|
||||
if response.status_code >= 400:
|
||||
raise RuntimeError(f"planner gateway returned {response.status_code}")
|
||||
data = response.json()
|
||||
content = (
|
||||
(data.get("choices") or [{}])[0].get("message", {}).get("content") or ""
|
||||
)
|
||||
parsed = _parse(content)
|
||||
if parsed:
|
||||
return parsed
|
||||
except Exception as exc:
|
||||
logger.warning("deepsearch query planner failed, using fallback: %s", exc)
|
||||
return _fallback(query)
|
||||
@@ -0,0 +1,216 @@
|
||||
# retoor <retoor@molodetz.nl>
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
import re
|
||||
from dataclasses import dataclass, field
|
||||
from urllib.parse import urlparse
|
||||
|
||||
import httpx
|
||||
|
||||
from devplacepy.config import INTERNAL_GATEWAY_URL, INTERNAL_MODEL
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
AGENT_TIMEOUT_SECONDS = 120.0
|
||||
SUMMARY_MAX_TOKENS = 900
|
||||
CRITIC_MAX_TOKENS = 600
|
||||
LINKER_MAX_TOKENS = 600
|
||||
MAX_CONTEXT_CHARS = 11000
|
||||
SCORE_MAX = 100
|
||||
CONFIDENCE_BASELINE = 0.35
|
||||
DIVERSITY_PAGES_PER_DOMAIN = 2.0
|
||||
|
||||
|
||||
@dataclass
|
||||
class Orchestration:
|
||||
summary: str = ""
|
||||
findings: list[dict] = field(default_factory=list)
|
||||
gaps: list[str] = field(default_factory=list)
|
||||
confidence: float = 0.0
|
||||
source_diversity: float = 0.0
|
||||
score: int = 0
|
||||
|
||||
|
||||
def _domain(url: str) -> str:
|
||||
try:
|
||||
return urlparse(url).netloc.lower()
|
||||
except ValueError:
|
||||
return ""
|
||||
|
||||
|
||||
def source_diversity(pages: list) -> float:
|
||||
if not pages:
|
||||
return 0.0
|
||||
domains = {_domain(page.url) for page in pages if getattr(page, "url", "")}
|
||||
domains.discard("")
|
||||
if not domains:
|
||||
return 0.0
|
||||
ratio = len(domains) / max(1.0, len(pages) / DIVERSITY_PAGES_PER_DOMAIN)
|
||||
return round(min(1.0, ratio), 3)
|
||||
|
||||
|
||||
def _build_context(pages: list) -> str:
|
||||
blocks: list[str] = []
|
||||
used = 0
|
||||
for index, page in enumerate(pages, start=1):
|
||||
snippet = (page.text or "")[:1600]
|
||||
block = f"[{index}] {page.title} ({page.url})\n{snippet}"
|
||||
if used + len(block) > MAX_CONTEXT_CHARS and blocks:
|
||||
break
|
||||
used += len(block)
|
||||
blocks.append(block)
|
||||
return "\n\n".join(blocks)
|
||||
|
||||
|
||||
async def _complete(messages: list[dict], api_key: str, max_tokens: int) -> str:
|
||||
payload = {
|
||||
"model": INTERNAL_MODEL,
|
||||
"messages": messages,
|
||||
"max_tokens": max_tokens,
|
||||
"temperature": 0.2,
|
||||
}
|
||||
headers = {"Authorization": f"Bearer {api_key}", "Content-Type": "application/json"}
|
||||
async with httpx.AsyncClient(timeout=AGENT_TIMEOUT_SECONDS) as client:
|
||||
response = await client.post(INTERNAL_GATEWAY_URL, json=payload, headers=headers)
|
||||
if response.status_code >= 400:
|
||||
raise RuntimeError(f"agent gateway returned {response.status_code}")
|
||||
data = response.json()
|
||||
return (data.get("choices") or [{}])[0].get("message", {}).get("content") or ""
|
||||
|
||||
|
||||
def _parse_json(text: str) -> dict:
|
||||
match = re.search(r"\{.*\}", text, re.DOTALL)
|
||||
if not match:
|
||||
return {}
|
||||
try:
|
||||
return json.loads(match.group())
|
||||
except (ValueError, TypeError):
|
||||
return {}
|
||||
|
||||
|
||||
SUMMARIZER_PROMPT = (
|
||||
"You are a research summarizer. Using ONLY the numbered SOURCES, write a JSON object "
|
||||
"with keys: 'summary' (a grounded markdown summary answering the question) and "
|
||||
"'findings' (an array of objects, each with 'title', 'detail', 'confidence' between 0 "
|
||||
"and 1, and 'citations' an array of source numbers). Cite only the provided sources. "
|
||||
"Return ONLY the JSON object."
|
||||
)
|
||||
CRITIC_PROMPT = (
|
||||
"You are a research critic. Given a QUESTION, a draft SUMMARY and FINDINGS, identify "
|
||||
"what is missing, contradictory, or weakly supported. Return ONLY a JSON object with "
|
||||
"key 'gaps': an array of short strings describing open questions or weak spots."
|
||||
)
|
||||
LINKER_PROMPT = (
|
||||
"You are a research linker. Given FINDINGS and the SOURCES, refine the confidence of "
|
||||
"each finding based on how many independent sources support it. Return ONLY a JSON "
|
||||
"object with key 'confidence': a number between 0 and 1 estimating overall answer "
|
||||
"confidence given source agreement and coverage."
|
||||
)
|
||||
|
||||
|
||||
def _heuristic(question: str, pages: list) -> Orchestration:
|
||||
diversity = source_diversity(pages)
|
||||
findings = []
|
||||
for page in pages[:5]:
|
||||
findings.append(
|
||||
{
|
||||
"title": page.title[:120] or page.url,
|
||||
"detail": (page.text or "")[:400],
|
||||
"confidence": round(min(0.6, CONFIDENCE_BASELINE + diversity / 4), 3),
|
||||
"citations": [page.url],
|
||||
}
|
||||
)
|
||||
summary = (
|
||||
f"Collected {len(pages)} sources for '{question}'. Automatic synthesis was "
|
||||
"unavailable, so the top findings are listed verbatim from the gathered sources."
|
||||
)
|
||||
confidence = round(min(0.6, CONFIDENCE_BASELINE + diversity / 3), 3)
|
||||
score = int(min(SCORE_MAX, len(pages) * 6 + diversity * 30))
|
||||
return Orchestration(
|
||||
summary=summary,
|
||||
findings=findings,
|
||||
gaps=["Automatic critique was unavailable for this run."],
|
||||
confidence=confidence,
|
||||
source_diversity=diversity,
|
||||
score=score,
|
||||
)
|
||||
|
||||
|
||||
async def orchestrate(question: str, pages: list, api_key: str, emit) -> Orchestration:
|
||||
diversity = source_diversity(pages)
|
||||
if not pages:
|
||||
return Orchestration(gaps=["No sources were gathered."], source_diversity=0.0)
|
||||
context = _build_context(pages)
|
||||
try:
|
||||
emit({"type": "agent", "agent": "summarizer", "message": "Synthesising findings"})
|
||||
summary_raw = await _complete(
|
||||
[
|
||||
{"role": "system", "content": SUMMARIZER_PROMPT},
|
||||
{
|
||||
"role": "user",
|
||||
"content": f"QUESTION: {question}\n\nSOURCES:\n{context}",
|
||||
},
|
||||
],
|
||||
api_key,
|
||||
SUMMARY_MAX_TOKENS,
|
||||
)
|
||||
parsed = _parse_json(summary_raw)
|
||||
summary = str(parsed.get("summary", "")).strip()
|
||||
findings = [f for f in (parsed.get("findings") or []) if isinstance(f, dict)]
|
||||
if not summary and not findings:
|
||||
return _heuristic(question, pages)
|
||||
|
||||
emit({"type": "agent", "agent": "critic", "message": "Reviewing for gaps"})
|
||||
gaps_raw = await _complete(
|
||||
[
|
||||
{"role": "system", "content": CRITIC_PROMPT},
|
||||
{
|
||||
"role": "user",
|
||||
"content": (
|
||||
f"QUESTION: {question}\n\nSUMMARY: {summary}\n\n"
|
||||
f"FINDINGS: {json.dumps(findings)[:4000]}"
|
||||
),
|
||||
},
|
||||
],
|
||||
api_key,
|
||||
CRITIC_MAX_TOKENS,
|
||||
)
|
||||
gaps = [str(g).strip() for g in (_parse_json(gaps_raw).get("gaps") or []) if str(g).strip()]
|
||||
|
||||
emit({"type": "agent", "agent": "linker", "message": "Scoring confidence"})
|
||||
link_raw = await _complete(
|
||||
[
|
||||
{"role": "system", "content": LINKER_PROMPT},
|
||||
{
|
||||
"role": "user",
|
||||
"content": (
|
||||
f"FINDINGS: {json.dumps(findings)[:4000]}\n\nSOURCES:\n{context[:4000]}"
|
||||
),
|
||||
},
|
||||
],
|
||||
api_key,
|
||||
LINKER_MAX_TOKENS,
|
||||
)
|
||||
try:
|
||||
confidence = float(_parse_json(link_raw).get("confidence", 0.0))
|
||||
except (TypeError, ValueError):
|
||||
confidence = 0.0
|
||||
confidence = round(max(CONFIDENCE_BASELINE, min(1.0, confidence)), 3)
|
||||
coverage = min(1.0, len(pages) / 10.0)
|
||||
score = int(
|
||||
min(SCORE_MAX, (confidence * 0.5 + diversity * 0.3 + coverage * 0.2) * SCORE_MAX)
|
||||
)
|
||||
return Orchestration(
|
||||
summary=summary,
|
||||
findings=findings,
|
||||
gaps=gaps,
|
||||
confidence=confidence,
|
||||
source_diversity=diversity,
|
||||
score=score,
|
||||
)
|
||||
except Exception as exc:
|
||||
logger.warning("deepsearch orchestration failed, using heuristic: %s", exc)
|
||||
return _heuristic(question, pages)
|
||||
@@ -0,0 +1,43 @@
|
||||
# retoor <retoor@molodetz.nl>
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
|
||||
MAX_BUFFER = 2000
|
||||
|
||||
|
||||
class ProgressHub:
|
||||
def __init__(self) -> None:
|
||||
self._subscribers: dict[str, set[asyncio.Queue]] = {}
|
||||
self._buffers: dict[str, list[dict]] = {}
|
||||
|
||||
def subscribe(self, uid: str) -> asyncio.Queue:
|
||||
queue: asyncio.Queue = asyncio.Queue()
|
||||
self._subscribers.setdefault(uid, set()).add(queue)
|
||||
return queue
|
||||
|
||||
def unsubscribe(self, uid: str, queue: asyncio.Queue) -> None:
|
||||
listeners = self._subscribers.get(uid)
|
||||
if not listeners:
|
||||
return
|
||||
listeners.discard(queue)
|
||||
if not listeners:
|
||||
self._subscribers.pop(uid, None)
|
||||
|
||||
def publish(self, uid: str, frame: dict) -> None:
|
||||
buffer = self._buffers.setdefault(uid, [])
|
||||
buffer.append(frame)
|
||||
if len(buffer) > MAX_BUFFER:
|
||||
del buffer[: len(buffer) - MAX_BUFFER]
|
||||
for queue in self._subscribers.get(uid, set()):
|
||||
queue.put_nowait(frame)
|
||||
|
||||
def snapshot(self, uid: str) -> list[dict]:
|
||||
return list(self._buffers.get(uid, []))
|
||||
|
||||
def clear(self, uid: str) -> None:
|
||||
self._buffers.pop(uid, None)
|
||||
|
||||
|
||||
hub = ProgressHub()
|
||||
@@ -0,0 +1,212 @@
|
||||
# retoor <retoor@molodetz.nl>
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import logging
|
||||
import shutil
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
from devplacepy.config import BASE_DIR, DEEPSEARCH_DIR
|
||||
from devplacepy.services.deepsearch.store import VectorStore
|
||||
from devplacepy.services.jobs.base import JobService
|
||||
|
||||
from .progress import hub
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
WORKER_MODULE = "devplacepy.services.jobs.deepsearch.worker"
|
||||
STREAM_LIMIT = 16 * 1024 * 1024
|
||||
|
||||
|
||||
class DeepsearchService(JobService):
|
||||
kind = "deepsearch"
|
||||
title = "DeepSearch"
|
||||
description = (
|
||||
"Runs a multi-agent web research job: it plans queries, crawls and indexes "
|
||||
"sources into a per-session vector collection, then synthesises a cited report "
|
||||
"with confidence scoring, source diversity and gap analysis, streaming live "
|
||||
"progress over a websocket."
|
||||
)
|
||||
|
||||
def __init__(self):
|
||||
super().__init__(name="deepsearch", interval_seconds=2)
|
||||
|
||||
def session_dir(self, uid: str) -> Path:
|
||||
return DEEPSEARCH_DIR / uid
|
||||
|
||||
def collection_name(self, uid: str) -> str:
|
||||
return f"ds_{uid.replace('-', '')}"
|
||||
|
||||
async def process(self, job: dict) -> dict:
|
||||
from devplacepy import database
|
||||
from devplacepy.services.audit import record as audit
|
||||
|
||||
uid = job["uid"]
|
||||
payload = dict(job.get("payload", {}))
|
||||
query = payload.get("query", "")
|
||||
output_dir = self.session_dir(uid)
|
||||
output_dir.mkdir(parents=True, exist_ok=True)
|
||||
(output_dir / "control.json").write_text(
|
||||
json.dumps({"state": "running"}), encoding="utf-8"
|
||||
)
|
||||
payload["collection"] = self.collection_name(uid)
|
||||
payload["cached_hashes"] = self._cached_hashes(database)
|
||||
payload_path = output_dir / "payload.json"
|
||||
payload_path.write_text(json.dumps(payload), encoding="utf-8")
|
||||
|
||||
actor_kind = job.get("owner_kind") or "system"
|
||||
actor_uid = job.get("owner_id") if job.get("owner_kind") == "user" else None
|
||||
database.update_deepsearch_session(uid, {"status": "running"})
|
||||
|
||||
try:
|
||||
summary = await self._run_worker(uid, payload_path, output_dir)
|
||||
except Exception as exc:
|
||||
hub.publish(uid, {"type": "failed", "message": str(exc)[:300]})
|
||||
database.update_deepsearch_session(uid, {"status": "failed"})
|
||||
audit.record_system(
|
||||
"deepsearch.run.failed",
|
||||
actor_kind=actor_kind,
|
||||
actor_uid=actor_uid,
|
||||
result="failure",
|
||||
summary=f"DeepSearch for {query} failed",
|
||||
metadata={"query": query, "error": str(exc)[:200]},
|
||||
links=[audit.job(uid)],
|
||||
)
|
||||
raise
|
||||
|
||||
report = self._load_report(output_dir)
|
||||
self._persist_cache(database, output_dir)
|
||||
from datetime import datetime, timezone
|
||||
|
||||
database.update_deepsearch_session(
|
||||
uid,
|
||||
{
|
||||
"status": "done",
|
||||
"score": int(summary.get("score") or 0),
|
||||
"confidence": float(summary.get("confidence") or 0.0),
|
||||
"source_diversity": float(summary.get("source_diversity") or 0.0),
|
||||
"page_count": int(summary.get("page_count") or 0),
|
||||
"chunk_count": int(summary.get("chunk_count") or 0),
|
||||
"summary": (report.get("summary") or "")[:4000],
|
||||
"completed_at": datetime.now(timezone.utc).isoformat(),
|
||||
},
|
||||
)
|
||||
hub.publish(
|
||||
uid,
|
||||
{
|
||||
"type": "done",
|
||||
"score": summary.get("score"),
|
||||
"confidence": summary.get("confidence"),
|
||||
"source_diversity": summary.get("source_diversity"),
|
||||
"page_count": summary.get("page_count"),
|
||||
"chunk_count": summary.get("chunk_count"),
|
||||
"session_url": f"/tools/deepsearch/{uid}/session",
|
||||
},
|
||||
)
|
||||
hub.clear(uid)
|
||||
audit.record_system(
|
||||
"deepsearch.run.complete",
|
||||
actor_kind=actor_kind,
|
||||
actor_uid=actor_uid,
|
||||
summary=f"DeepSearch for {query} scored {summary.get('score')}",
|
||||
metadata={
|
||||
"query": query,
|
||||
"score": summary.get("score"),
|
||||
"page_count": summary.get("page_count"),
|
||||
},
|
||||
links=[audit.job(uid)],
|
||||
)
|
||||
return {
|
||||
"query": query,
|
||||
"score": summary.get("score", 0),
|
||||
"confidence": summary.get("confidence", 0.0),
|
||||
"source_diversity": summary.get("source_diversity", 0.0),
|
||||
"page_count": summary.get("page_count", 0),
|
||||
"chunk_count": summary.get("chunk_count", 0),
|
||||
"report": report,
|
||||
"collection": self.collection_name(uid),
|
||||
"session_url": f"/tools/deepsearch/{uid}/session",
|
||||
"bytes_in": 0,
|
||||
"bytes_out": len(json.dumps(report)) if report else 0,
|
||||
"item_count": summary.get("page_count", 0),
|
||||
}
|
||||
|
||||
def _cached_hashes(self, database) -> list[str]:
|
||||
if "deepsearch_url_cache" not in database.db.tables:
|
||||
return []
|
||||
return [
|
||||
row.get("url_hash", "")
|
||||
for row in database.get_table("deepsearch_url_cache").find()
|
||||
if row.get("url_hash")
|
||||
]
|
||||
|
||||
def _persist_cache(self, database, output_dir: Path) -> None:
|
||||
path = output_dir / "url_cache.json"
|
||||
if not path.is_file():
|
||||
return
|
||||
try:
|
||||
entries = json.loads(path.read_text(encoding="utf-8"))
|
||||
except (ValueError, OSError):
|
||||
return
|
||||
for entry in entries:
|
||||
database.upsert_deepsearch_url_cache(
|
||||
entry.get("url_hash", ""),
|
||||
entry.get("url", ""),
|
||||
entry.get("title", ""),
|
||||
entry.get("content_hash", ""),
|
||||
int(entry.get("status") or 0),
|
||||
int(entry.get("byte_size") or 0),
|
||||
)
|
||||
|
||||
async def _run_worker(self, uid: str, payload_path: Path, output_dir: Path) -> dict:
|
||||
proc = await asyncio.create_subprocess_exec(
|
||||
sys.executable,
|
||||
"-m",
|
||||
WORKER_MODULE,
|
||||
str(payload_path),
|
||||
str(output_dir),
|
||||
cwd=str(BASE_DIR),
|
||||
stdout=asyncio.subprocess.PIPE,
|
||||
stderr=asyncio.subprocess.PIPE,
|
||||
limit=STREAM_LIMIT,
|
||||
)
|
||||
summary: dict = {}
|
||||
worker_error = ""
|
||||
while True:
|
||||
line = await proc.stdout.readline()
|
||||
if not line:
|
||||
break
|
||||
try:
|
||||
frame = json.loads(line.decode("utf-8", "replace"))
|
||||
except (ValueError, TypeError):
|
||||
continue
|
||||
hub.publish(uid, frame)
|
||||
if frame.get("type") == "report_ready":
|
||||
summary = frame
|
||||
elif frame.get("type") == "error":
|
||||
worker_error = frame.get("message", "worker error")
|
||||
err = (await proc.stderr.read()).decode("utf-8", "replace")
|
||||
await proc.wait()
|
||||
if proc.returncode != 0 or not summary:
|
||||
raise RuntimeError(
|
||||
worker_error or err[:500] or f"deepsearch worker exited {proc.returncode}"
|
||||
)
|
||||
return summary
|
||||
|
||||
def _load_report(self, output_dir: Path) -> dict:
|
||||
report_path = output_dir / "report.json"
|
||||
if not report_path.is_file():
|
||||
return {}
|
||||
try:
|
||||
return json.loads(report_path.read_text(encoding="utf-8"))
|
||||
except (ValueError, OSError):
|
||||
return {}
|
||||
|
||||
def cleanup(self, job: dict) -> None:
|
||||
uid = job["uid"]
|
||||
hub.clear(uid)
|
||||
VectorStore(self.collection_name(uid)).drop()
|
||||
shutil.rmtree(self.session_dir(uid), ignore_errors=True)
|
||||
@@ -0,0 +1,186 @@
|
||||
# retoor <retoor@molodetz.nl>
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import sys
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
|
||||
from devplacepy.services.deepsearch.embeddings import embed_texts, local_embed
|
||||
from devplacepy.services.deepsearch.store import Chunk, VectorStore
|
||||
from devplacepy.utils import generate_uid
|
||||
|
||||
from .chunking import chunk_text
|
||||
from .crawl import content_hash, crawl, search_queries, url_hash
|
||||
from .enhance import plan_queries
|
||||
from .orchestrate import orchestrate
|
||||
|
||||
CONTROL_FILE = "control.json"
|
||||
PAUSE_POLL_SECONDS = 1.0
|
||||
EMBED_BATCH = 64
|
||||
|
||||
|
||||
def _emit(frame: dict) -> None:
|
||||
sys.stdout.write(json.dumps(frame, ensure_ascii=False) + "\n")
|
||||
sys.stdout.flush()
|
||||
|
||||
|
||||
def _read_control(output_dir: Path) -> str:
|
||||
path = output_dir / CONTROL_FILE
|
||||
if not path.is_file():
|
||||
return "running"
|
||||
try:
|
||||
return str(json.loads(path.read_text(encoding="utf-8")).get("state", "running"))
|
||||
except (ValueError, OSError):
|
||||
return "running"
|
||||
|
||||
|
||||
def _make_stop(output_dir: Path):
|
||||
async def should_stop() -> bool:
|
||||
while True:
|
||||
state = _read_control(output_dir)
|
||||
if state == "cancelled":
|
||||
return True
|
||||
if state == "paused":
|
||||
_emit({"type": "stage", "stage": "paused", "message": "Paused"})
|
||||
await asyncio.sleep(PAUSE_POLL_SECONDS)
|
||||
continue
|
||||
return False
|
||||
|
||||
return should_stop
|
||||
|
||||
|
||||
async def _index_chunks(
|
||||
store: VectorStore, pages: list, api_key: str
|
||||
) -> tuple[int, str]:
|
||||
chunks: list[Chunk] = []
|
||||
for page in pages:
|
||||
for position, text in enumerate(chunk_text(page.text)):
|
||||
chunks.append(
|
||||
Chunk(
|
||||
uid=generate_uid(),
|
||||
text=text,
|
||||
url=page.url,
|
||||
title=page.title,
|
||||
depth=page.depth,
|
||||
source=page.source,
|
||||
position=position,
|
||||
)
|
||||
)
|
||||
if not chunks:
|
||||
return 0, "empty"
|
||||
backend = "gateway"
|
||||
for start in range(0, len(chunks), EMBED_BATCH):
|
||||
batch = chunks[start : start + EMBED_BATCH]
|
||||
result = await embed_texts([chunk.text for chunk in batch], api_key)
|
||||
if not result.vectors:
|
||||
result = local_embed([chunk.text for chunk in batch])
|
||||
backend = result.backend
|
||||
store.add(batch, result.vectors)
|
||||
return len(chunks), backend
|
||||
|
||||
|
||||
async def _run(payload: dict, output_dir: Path) -> dict:
|
||||
query = payload.get("query", "")
|
||||
max_pages = int(payload.get("max_pages", 12))
|
||||
depth = int(payload.get("depth", 2))
|
||||
api_key = payload.get("api_key", "")
|
||||
collection = payload.get("collection", "")
|
||||
cached_hashes = set(payload.get("cached_hashes", []))
|
||||
should_stop = _make_stop(output_dir)
|
||||
|
||||
_emit({"type": "stage", "stage": "planning", "message": "Planning research queries"})
|
||||
queries = await plan_queries(query, api_key)
|
||||
_emit({"type": "queries", "queries": queries})
|
||||
|
||||
_emit({"type": "stage", "stage": "searching", "message": "Searching the web"})
|
||||
candidates = await search_queries(queries)
|
||||
_emit({"type": "candidates", "count": len(candidates)})
|
||||
|
||||
_emit({"type": "stage", "stage": "crawling", "message": "Crawling sources"})
|
||||
outcome = await crawl(
|
||||
candidates,
|
||||
max_pages,
|
||||
_emit,
|
||||
lambda url: url_hash(url) in cached_hashes,
|
||||
should_stop,
|
||||
)
|
||||
|
||||
new_cache = [
|
||||
{
|
||||
"url_hash": url_hash(page.url),
|
||||
"url": page.url,
|
||||
"title": page.title,
|
||||
"content_hash": content_hash(page.text),
|
||||
"status": page.status,
|
||||
"byte_size": len(page.text),
|
||||
}
|
||||
for page in outcome.pages
|
||||
]
|
||||
|
||||
store = VectorStore(collection)
|
||||
_emit({"type": "stage", "stage": "indexing", "message": "Indexing content"})
|
||||
chunk_count, embed_backend = await _index_chunks(store, outcome.pages, api_key)
|
||||
|
||||
_emit({"type": "stage", "stage": "analysis", "message": "Running research agents"})
|
||||
result = await orchestrate(query, outcome.pages, api_key, _emit)
|
||||
|
||||
sources = [
|
||||
{"url": page.url, "title": page.title, "source": page.source}
|
||||
for page in outcome.pages
|
||||
]
|
||||
report = {
|
||||
"query": query,
|
||||
"depth": depth,
|
||||
"max_pages": max_pages,
|
||||
"generated_at": datetime.now(timezone.utc).isoformat(),
|
||||
"summary": result.summary,
|
||||
"findings": result.findings,
|
||||
"gaps": result.gaps,
|
||||
"sources": sources,
|
||||
"score": result.score,
|
||||
"confidence": result.confidence,
|
||||
"source_diversity": result.source_diversity,
|
||||
"page_count": len(outcome.pages),
|
||||
"chunk_count": chunk_count,
|
||||
"embed_backend": embed_backend,
|
||||
"collection": collection,
|
||||
}
|
||||
(output_dir / "report.json").write_text(
|
||||
json.dumps(report, ensure_ascii=False), encoding="utf-8"
|
||||
)
|
||||
(output_dir / "url_cache.json").write_text(
|
||||
json.dumps(new_cache, ensure_ascii=False), encoding="utf-8"
|
||||
)
|
||||
_emit(
|
||||
{
|
||||
"type": "report_ready",
|
||||
"score": report["score"],
|
||||
"confidence": report["confidence"],
|
||||
"source_diversity": report["source_diversity"],
|
||||
"page_count": report["page_count"],
|
||||
"chunk_count": report["chunk_count"],
|
||||
}
|
||||
)
|
||||
return report
|
||||
|
||||
|
||||
def main(argv: list) -> int:
|
||||
if len(argv) != 3:
|
||||
sys.stderr.write("usage: deepsearch.worker <payload_json> <output_dir>\n")
|
||||
return 2
|
||||
payload = json.loads(Path(argv[1]).read_text(encoding="utf-8"))
|
||||
output_dir = Path(argv[2])
|
||||
output_dir.mkdir(parents=True, exist_ok=True)
|
||||
try:
|
||||
asyncio.run(_run(payload, output_dir))
|
||||
except Exception as exc:
|
||||
_emit({"type": "error", "message": str(exc)[:500]})
|
||||
return 1
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main(sys.argv))
|
||||
@@ -6,6 +6,8 @@ TIMEOUT_DEFAULT = 300
|
||||
TIMEOUT_MIN = 300
|
||||
INSTANCES_DEFAULT = 4
|
||||
|
||||
SYSTEM_PREAMBLE_DEFAULT = ""
|
||||
|
||||
VISION_URL_DEFAULT = "https://openrouter.ai/api/v1/chat/completions"
|
||||
VISION_MODEL_DEFAULT = "google/gemma-3-12b-it"
|
||||
VISION_CACHE_SIZE_DEFAULT = 256
|
||||
|
||||
@@ -12,6 +12,7 @@ from fastapi.responses import JSONResponse, Response, StreamingResponse
|
||||
|
||||
from devplacepy.services.openai_gateway import config
|
||||
from devplacepy.services.openai_gateway.reliability import CircuitBreaker, retry_send
|
||||
from devplacepy.services.openai_gateway.system_message import apply_system_directives
|
||||
from devplacepy.services.openai_gateway.usage import (
|
||||
GatewayUsageLedger,
|
||||
classify_error,
|
||||
@@ -229,6 +230,8 @@ class GatewayRuntime:
|
||||
messages = await augmenter.augment_messages(client, messages)
|
||||
self.vision_calls += augmenter.calls
|
||||
|
||||
messages = apply_system_directives(messages, cfg.get("gateway_system_preamble", ""))
|
||||
|
||||
requested = body.get("model")
|
||||
if cfg["gateway_force_model"] or not requested or requested == "molodetz":
|
||||
model = cfg["gateway_model"]
|
||||
@@ -351,6 +354,25 @@ class GatewayRuntime:
|
||||
):
|
||||
log = log or (lambda message: None)
|
||||
if not cfg["gateway_embed_enabled"]:
|
||||
from devplacepy.services.audit import record as audit
|
||||
from devplacepy.services.openai_gateway.usage import audit_actor_for
|
||||
|
||||
actor_kind, actor_uid, actor_role = audit_actor_for(owner[0], owner[1])
|
||||
audit.record_system(
|
||||
"ai.gateway.call",
|
||||
actor_kind=actor_kind,
|
||||
actor_uid=actor_uid,
|
||||
actor_role=actor_role,
|
||||
origin="api",
|
||||
result="denied",
|
||||
summary="embeddings disabled",
|
||||
metadata={
|
||||
"backend": "embed",
|
||||
"endpoint": "embeddings",
|
||||
"owner_kind": owner[0],
|
||||
"owner_id": owner[1],
|
||||
},
|
||||
)
|
||||
return JSONResponse(
|
||||
status_code=503,
|
||||
content={
|
||||
|
||||
@@ -89,6 +89,16 @@ class GatewayService(BaseService):
|
||||
help="Max concurrent upstream forwards per worker (connection pool + semaphore).",
|
||||
group="Upstream",
|
||||
),
|
||||
ConfigField(
|
||||
"gateway_system_preamble",
|
||||
"System preamble",
|
||||
type="text",
|
||||
default=config.SYSTEM_PREAMBLE_DEFAULT,
|
||||
help="Operator text prepended ahead of every chat request's system message "
|
||||
"(before an auto-injected EU-format date line and the client's own system "
|
||||
"content, all in one system message). Leave blank to disable.",
|
||||
group="Prompt",
|
||||
),
|
||||
ConfigField(
|
||||
"gateway_vision_enabled",
|
||||
"Vision augmentation",
|
||||
|
||||
@@ -0,0 +1,110 @@
|
||||
# retoor <retoor@molodetz.nl>
|
||||
|
||||
import logging
|
||||
import re
|
||||
from datetime import datetime
|
||||
from typing import Any, Optional
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
DATE_LABEL = "Current date"
|
||||
|
||||
_MONTH_NAMES = (
|
||||
"january|february|march|april|may|june|july|august|september|october|"
|
||||
"november|december|jan|feb|mar|apr|jun|jul|aug|sep|sept|oct|nov|dec"
|
||||
)
|
||||
|
||||
_DATE_PATTERNS: tuple[re.Pattern[str], ...] = (
|
||||
re.compile(r"\b\d{4}-\d{1,2}-\d{1,2}\b"),
|
||||
re.compile(r"\b\d{1,2}/\d{1,2}/\d{4}\b"),
|
||||
re.compile(r"\b\d{1,2}-\d{1,2}-\d{4}\b"),
|
||||
re.compile(r"\b\d{1,2}\.\d{1,2}\.\d{4}\b"),
|
||||
re.compile(
|
||||
rf"\b\d{{1,2}}\s+(?:{_MONTH_NAMES})\.?\s+\d{{4}}\b",
|
||||
re.IGNORECASE,
|
||||
),
|
||||
re.compile(
|
||||
rf"\b(?:{_MONTH_NAMES})\.?\s+\d{{1,2}}(?:st|nd|rd|th)?,?\s+\d{{4}}\b",
|
||||
re.IGNORECASE,
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def contains_date(text: str) -> bool:
|
||||
return any(pattern.search(text) for pattern in _DATE_PATTERNS)
|
||||
|
||||
|
||||
def current_date_eu() -> str:
|
||||
return datetime.now().strftime("%d/%m/%Y")
|
||||
|
||||
|
||||
def system_message_text(content: Any) -> str:
|
||||
if isinstance(content, str):
|
||||
return content
|
||||
if isinstance(content, list):
|
||||
parts: list[str] = []
|
||||
for block in content:
|
||||
if isinstance(block, dict) and block.get("type") == "text":
|
||||
value = block.get("text")
|
||||
if isinstance(value, str):
|
||||
parts.append(value)
|
||||
return "\n".join(parts)
|
||||
return ""
|
||||
|
||||
|
||||
def compose_system_content(
|
||||
preamble: str, client_content: Any, date_eu: str
|
||||
) -> str:
|
||||
preamble_text = (preamble or "").strip()
|
||||
client_text = system_message_text(client_content)
|
||||
sections: list[str] = []
|
||||
if preamble_text:
|
||||
sections.append(preamble_text)
|
||||
combined_for_date = f"{preamble_text}\n{client_text}"
|
||||
if not contains_date(combined_for_date):
|
||||
sections.append(f"{DATE_LABEL}: {date_eu}")
|
||||
if client_text:
|
||||
sections.append(client_text)
|
||||
return "\n\n".join(sections)
|
||||
|
||||
|
||||
def apply_system_directives(
|
||||
messages: list, preamble: str, date_eu: Optional[str] = None
|
||||
) -> list:
|
||||
date_value = date_eu or current_date_eu()
|
||||
preamble_text = (preamble or "").strip()
|
||||
result: list = list(messages)
|
||||
system_index = next(
|
||||
(
|
||||
index
|
||||
for index, message in enumerate(result)
|
||||
if isinstance(message, dict) and message.get("role") == "system"
|
||||
),
|
||||
None,
|
||||
)
|
||||
if system_index is not None:
|
||||
original = result[system_index]
|
||||
composed = compose_system_content(
|
||||
preamble_text, original.get("content"), date_value
|
||||
)
|
||||
updated = dict(original)
|
||||
updated["content"] = composed
|
||||
result[system_index] = updated
|
||||
logger.debug(
|
||||
"Gateway composed existing system message (preamble=%s, date=%s)",
|
||||
bool(preamble_text),
|
||||
date_value,
|
||||
)
|
||||
return result
|
||||
if not preamble_text:
|
||||
return result
|
||||
composed = compose_system_content(preamble_text, "", date_value)
|
||||
if not composed.strip():
|
||||
return result
|
||||
result.insert(0, {"role": "system", "content": composed})
|
||||
logger.info(
|
||||
"Gateway injected a system message (preamble=%s, date=%s)",
|
||||
bool(preamble_text),
|
||||
date_value,
|
||||
)
|
||||
return result
|
||||
@@ -204,6 +204,21 @@ def classify_error(
|
||||
return "gateway"
|
||||
|
||||
|
||||
def audit_actor_for(owner_kind: str, owner_id: str) -> tuple[str, Optional[str], str]:
|
||||
actor_kind = (
|
||||
"guest"
|
||||
if owner_kind == "guest"
|
||||
else ("user" if owner_kind in ("user", "admin") else "system")
|
||||
)
|
||||
actor_uid = owner_id if actor_kind == "user" else None
|
||||
actor_role = (
|
||||
"admin"
|
||||
if owner_kind == "admin"
|
||||
else (actor_kind if actor_kind != "user" else "member")
|
||||
)
|
||||
return actor_kind, actor_uid, actor_role
|
||||
|
||||
|
||||
class GatewayUsageLedger:
|
||||
def record(self, raw: dict, pricing: Pricing, context_map: dict) -> None:
|
||||
try:
|
||||
@@ -271,12 +286,12 @@ class GatewayUsageLedger:
|
||||
|
||||
owner_kind = raw.get("owner_kind") or "unknown"
|
||||
owner_id = raw.get("owner_id") or "unknown"
|
||||
actor_kind = "guest" if owner_kind == "guest" else ("user" if owner_kind in ("user", "admin") else "system")
|
||||
actor_kind, actor_uid, actor_role = audit_actor_for(owner_kind, owner_id)
|
||||
audit.record_system(
|
||||
"ai.gateway.call",
|
||||
actor_kind=actor_kind,
|
||||
actor_uid=owner_id if actor_kind == "user" else None,
|
||||
actor_role="admin" if owner_kind == "admin" else (actor_kind if actor_kind != "user" else "member"),
|
||||
actor_uid=actor_uid,
|
||||
actor_role=actor_role,
|
||||
origin="api",
|
||||
result="success" if raw.get("success") else "failure",
|
||||
summary=f"LLM call by {owner_kind}/{owner_id} (model {raw.get('model') or ''})",
|
||||
|
||||
@@ -0,0 +1,417 @@
|
||||
/* retoor <retoor@molodetz.nl> */
|
||||
|
||||
.ds-layout {
|
||||
display: grid;
|
||||
grid-template-columns: var(--sidebar-width) 1fr;
|
||||
gap: var(--space-xl);
|
||||
align-items: start;
|
||||
}
|
||||
|
||||
.ds-main {
|
||||
min-width: 0;
|
||||
}
|
||||
|
||||
.ds-side-list {
|
||||
list-style: none;
|
||||
margin: 0;
|
||||
padding: 0;
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
gap: var(--space-sm);
|
||||
font-size: 0.8125rem;
|
||||
color: var(--text-secondary);
|
||||
line-height: 1.5;
|
||||
}
|
||||
|
||||
.ds-side-list li {
|
||||
padding-left: var(--space-md);
|
||||
position: relative;
|
||||
}
|
||||
|
||||
.ds-side-list li::before {
|
||||
content: "";
|
||||
position: absolute;
|
||||
left: 0;
|
||||
top: 0.55em;
|
||||
width: 5px;
|
||||
height: 5px;
|
||||
border-radius: 50%;
|
||||
background: var(--accent);
|
||||
}
|
||||
|
||||
.ds-header h1 {
|
||||
font-size: 1.75rem;
|
||||
margin: 0 0 0.5rem;
|
||||
}
|
||||
|
||||
.ds-header p {
|
||||
color: var(--text-secondary);
|
||||
margin: 0 0 var(--space-lg);
|
||||
}
|
||||
|
||||
.ds-form-card,
|
||||
.ds-card {
|
||||
background: var(--bg-card);
|
||||
border: 1px solid var(--border);
|
||||
border-radius: var(--radius-lg);
|
||||
box-shadow: var(--shadow-sm);
|
||||
padding: var(--space-xl);
|
||||
margin-bottom: var(--space-xl);
|
||||
}
|
||||
|
||||
.ds-input {
|
||||
width: 100%;
|
||||
background: var(--bg-input);
|
||||
border: 1px solid var(--border);
|
||||
border-radius: var(--radius-input);
|
||||
color: var(--text-primary);
|
||||
padding: var(--space-md);
|
||||
font-family: inherit;
|
||||
resize: vertical;
|
||||
}
|
||||
|
||||
.ds-form-row {
|
||||
display: flex;
|
||||
flex-wrap: wrap;
|
||||
align-items: flex-end;
|
||||
gap: var(--space-md);
|
||||
margin-top: var(--space-md);
|
||||
}
|
||||
|
||||
.ds-field {
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
gap: 4px;
|
||||
font-size: 0.8125rem;
|
||||
color: var(--text-secondary);
|
||||
}
|
||||
|
||||
.ds-input-num {
|
||||
width: 5rem;
|
||||
background: var(--bg-input);
|
||||
border: 1px solid var(--border);
|
||||
border-radius: var(--radius-input);
|
||||
color: var(--text-primary);
|
||||
padding: var(--space-sm);
|
||||
}
|
||||
|
||||
.ds-run-btn {
|
||||
margin-left: auto;
|
||||
}
|
||||
|
||||
.ds-form-error {
|
||||
color: var(--danger, #e5484d);
|
||||
margin: var(--space-md) 0 0;
|
||||
font-size: 0.875rem;
|
||||
}
|
||||
|
||||
.ds-live {
|
||||
background: var(--bg-card);
|
||||
border: 1px solid var(--border);
|
||||
border-radius: var(--radius-lg);
|
||||
padding: var(--space-lg);
|
||||
margin-bottom: var(--space-xl);
|
||||
}
|
||||
|
||||
.ds-live-head {
|
||||
display: flex;
|
||||
flex-wrap: wrap;
|
||||
align-items: center;
|
||||
gap: var(--space-md);
|
||||
margin-bottom: var(--space-md);
|
||||
}
|
||||
|
||||
.ds-live-status {
|
||||
font-weight: 600;
|
||||
color: var(--text-primary);
|
||||
}
|
||||
|
||||
.ds-live-count {
|
||||
color: var(--text-muted);
|
||||
font-size: 0.875rem;
|
||||
}
|
||||
|
||||
.ds-live-controls {
|
||||
margin-left: auto;
|
||||
display: flex;
|
||||
gap: var(--space-sm);
|
||||
}
|
||||
|
||||
.ds-ctl {
|
||||
background: var(--bg-card-hover);
|
||||
border: 1px solid var(--border);
|
||||
color: var(--text-secondary);
|
||||
padding: 4px 12px;
|
||||
font-size: 0.8125rem;
|
||||
}
|
||||
|
||||
.ds-progress {
|
||||
height: 6px;
|
||||
border-radius: 3px;
|
||||
background: var(--border);
|
||||
overflow: hidden;
|
||||
}
|
||||
|
||||
.ds-progress-bar {
|
||||
height: 100%;
|
||||
width: 0;
|
||||
background: var(--accent-gradient, var(--accent));
|
||||
transition: width 0.3s ease;
|
||||
}
|
||||
|
||||
.ds-timeline {
|
||||
list-style: none;
|
||||
margin: var(--space-md) 0 0;
|
||||
padding: 0;
|
||||
max-height: 300px;
|
||||
overflow-y: auto;
|
||||
font-size: 0.8125rem;
|
||||
color: var(--text-secondary);
|
||||
}
|
||||
|
||||
.ds-timeline li {
|
||||
padding: 4px 0;
|
||||
border-bottom: 1px solid var(--border);
|
||||
}
|
||||
|
||||
.ds-done {
|
||||
background: var(--bg-card);
|
||||
border: 1px solid var(--border);
|
||||
border-radius: var(--radius-lg);
|
||||
padding: var(--space-lg);
|
||||
text-align: center;
|
||||
}
|
||||
|
||||
.ds-session {
|
||||
display: grid;
|
||||
grid-template-columns: 1fr 360px;
|
||||
gap: var(--space-xl);
|
||||
align-items: start;
|
||||
}
|
||||
|
||||
.ds-session-main {
|
||||
min-width: 0;
|
||||
}
|
||||
|
||||
.ds-session-head h1 {
|
||||
font-size: 1.5rem;
|
||||
margin: 0 0 var(--space-md);
|
||||
}
|
||||
|
||||
.ds-metrics {
|
||||
display: flex;
|
||||
flex-wrap: wrap;
|
||||
gap: var(--space-md);
|
||||
margin-bottom: var(--space-md);
|
||||
}
|
||||
|
||||
.ds-metric {
|
||||
background: var(--bg-card);
|
||||
border: 1px solid var(--border);
|
||||
border-radius: var(--radius);
|
||||
padding: 4px 12px;
|
||||
font-size: 0.8125rem;
|
||||
color: var(--text-secondary);
|
||||
}
|
||||
|
||||
.ds-metric strong {
|
||||
color: var(--accent);
|
||||
}
|
||||
|
||||
.ds-exports {
|
||||
display: flex;
|
||||
gap: var(--space-sm);
|
||||
margin-bottom: var(--space-lg);
|
||||
}
|
||||
|
||||
.ds-export {
|
||||
background: var(--bg-card-hover);
|
||||
border: 1px solid var(--border);
|
||||
color: var(--text-secondary);
|
||||
padding: 6px 14px;
|
||||
font-size: 0.8125rem;
|
||||
}
|
||||
|
||||
.ds-card h2 {
|
||||
font-size: 1.125rem;
|
||||
margin: 0 0 var(--space-md);
|
||||
}
|
||||
|
||||
.ds-findings {
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
gap: var(--space-sm);
|
||||
}
|
||||
|
||||
.ds-finding {
|
||||
border: 1px solid var(--border);
|
||||
border-radius: var(--radius);
|
||||
padding: var(--space-sm) var(--space-md);
|
||||
}
|
||||
|
||||
.ds-finding summary {
|
||||
display: flex;
|
||||
justify-content: space-between;
|
||||
cursor: pointer;
|
||||
font-weight: 600;
|
||||
color: var(--text-primary);
|
||||
}
|
||||
|
||||
.ds-finding-confidence {
|
||||
color: var(--accent);
|
||||
font-size: 0.8125rem;
|
||||
}
|
||||
|
||||
.ds-finding-detail {
|
||||
margin-top: var(--space-sm);
|
||||
color: var(--text-secondary);
|
||||
line-height: 1.6;
|
||||
}
|
||||
|
||||
.ds-gaps {
|
||||
margin: 0;
|
||||
padding-left: var(--space-lg);
|
||||
color: var(--text-secondary);
|
||||
line-height: 1.6;
|
||||
}
|
||||
|
||||
.ds-sources {
|
||||
margin: 0;
|
||||
padding-left: var(--space-lg);
|
||||
color: var(--text-secondary);
|
||||
line-height: 1.8;
|
||||
}
|
||||
|
||||
.ds-source-tag {
|
||||
color: var(--text-muted);
|
||||
font-size: 0.75rem;
|
||||
margin-left: var(--space-sm);
|
||||
}
|
||||
|
||||
.ds-chat-pane {
|
||||
background: var(--bg-card);
|
||||
border: 1px solid var(--border);
|
||||
border-radius: var(--radius-lg);
|
||||
box-shadow: var(--shadow-sm);
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
position: sticky;
|
||||
top: var(--space-lg);
|
||||
max-height: calc(100vh - 120px);
|
||||
}
|
||||
|
||||
.ds-chat-head {
|
||||
padding: var(--space-md) var(--space-lg);
|
||||
border-bottom: 1px solid var(--border);
|
||||
font-weight: 600;
|
||||
}
|
||||
|
||||
.ds-chat-pending {
|
||||
padding: var(--space-lg);
|
||||
color: var(--text-muted);
|
||||
}
|
||||
|
||||
dp-deepsearch-chat {
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
min-height: 0;
|
||||
flex: 1;
|
||||
}
|
||||
|
||||
.ds-chat-log {
|
||||
flex: 1;
|
||||
overflow-y: auto;
|
||||
padding: var(--space-md);
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
gap: var(--space-md);
|
||||
min-height: 240px;
|
||||
}
|
||||
|
||||
.ds-chat-msg {
|
||||
display: flex;
|
||||
}
|
||||
|
||||
.ds-chat-msg.user {
|
||||
justify-content: flex-end;
|
||||
}
|
||||
|
||||
.ds-chat-bubble {
|
||||
max-width: 90%;
|
||||
padding: var(--space-sm) var(--space-md);
|
||||
border-radius: var(--radius);
|
||||
font-size: 0.875rem;
|
||||
line-height: 1.5;
|
||||
}
|
||||
|
||||
.ds-chat-msg.user .ds-chat-bubble {
|
||||
background: var(--accent);
|
||||
color: var(--white);
|
||||
}
|
||||
|
||||
.ds-chat-msg.agent .ds-chat-bubble {
|
||||
background: var(--bg-card-hover);
|
||||
color: var(--text-primary);
|
||||
}
|
||||
|
||||
.ds-chat-msg.error .ds-chat-bubble {
|
||||
background: var(--bg-card-hover);
|
||||
color: var(--danger, #e5484d);
|
||||
}
|
||||
|
||||
.ds-chat-citations {
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
gap: 2px;
|
||||
margin-top: var(--space-sm);
|
||||
font-size: 0.75rem;
|
||||
}
|
||||
|
||||
.ds-chat-citations a {
|
||||
color: var(--accent);
|
||||
}
|
||||
|
||||
.ds-chat-status {
|
||||
padding: 0 var(--space-md) var(--space-sm);
|
||||
font-size: 0.75rem;
|
||||
color: var(--text-muted);
|
||||
}
|
||||
|
||||
.ds-chat-input {
|
||||
display: flex;
|
||||
gap: var(--space-sm);
|
||||
padding: var(--space-md);
|
||||
border-top: 1px solid var(--border);
|
||||
}
|
||||
|
||||
.ds-chat-field {
|
||||
flex: 1;
|
||||
resize: none;
|
||||
background: var(--bg-input);
|
||||
border: 1px solid var(--border);
|
||||
border-radius: var(--radius-input);
|
||||
color: var(--text-primary);
|
||||
padding: var(--space-sm);
|
||||
font-family: inherit;
|
||||
}
|
||||
|
||||
.ds-chat-send {
|
||||
background: var(--accent);
|
||||
color: var(--white);
|
||||
border: none;
|
||||
border-radius: var(--radius-input);
|
||||
padding: 0 var(--space-md);
|
||||
cursor: pointer;
|
||||
}
|
||||
|
||||
@media (max-width: 900px) {
|
||||
.ds-layout,
|
||||
.ds-session {
|
||||
grid-template-columns: 1fr;
|
||||
}
|
||||
|
||||
.ds-chat-pane {
|
||||
position: static;
|
||||
max-height: 70vh;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,50 @@
|
||||
// retoor <retoor@molodetz.nl>
|
||||
|
||||
const WRONG_WORKER_CODE = 4013;
|
||||
const WRONG_WORKER_RETRY_MS = 200;
|
||||
|
||||
export class DeepsearchProgressSocket {
|
||||
constructor(uid, handlers) {
|
||||
this.handlers = handlers || {};
|
||||
this.ws = null;
|
||||
this._shouldRun = true;
|
||||
const scheme = location.protocol === "https:" ? "wss://" : "ws://";
|
||||
this.url = `${scheme}${location.host}/tools/deepsearch/${encodeURIComponent(uid)}/ws`;
|
||||
}
|
||||
|
||||
connect() {
|
||||
this._shouldRun = true;
|
||||
this._open();
|
||||
}
|
||||
|
||||
_open() {
|
||||
this.ws = new WebSocket(this.url);
|
||||
this.ws.addEventListener("message", (event) => {
|
||||
let frame;
|
||||
try {
|
||||
frame = JSON.parse(event.data);
|
||||
} catch {
|
||||
return;
|
||||
}
|
||||
this._emit("onMessage", frame);
|
||||
});
|
||||
this.ws.addEventListener("close", (event) => {
|
||||
if (event.code === WRONG_WORKER_CODE && this._shouldRun) {
|
||||
window.setTimeout(() => this._open(), WRONG_WORKER_RETRY_MS);
|
||||
return;
|
||||
}
|
||||
this._emit("onClose");
|
||||
});
|
||||
this.ws.addEventListener("error", () => this._emit("onError"));
|
||||
}
|
||||
|
||||
close() {
|
||||
this._shouldRun = false;
|
||||
if (this.ws) this.ws.close();
|
||||
}
|
||||
|
||||
_emit(name, payload) {
|
||||
const handler = this.handlers[name];
|
||||
if (typeof handler === "function") handler(payload);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,162 @@
|
||||
// retoor <retoor@molodetz.nl>
|
||||
|
||||
import { Http } from "./Http.js";
|
||||
import { DeepsearchProgressSocket } from "./DeepsearchProgressSocket.js";
|
||||
|
||||
export class DeepsearchTool {
|
||||
constructor() {
|
||||
this.root = document.querySelector("[data-deepsearch-tool]");
|
||||
if (!this.root) return;
|
||||
this.form = this.root.querySelector("[data-deepsearch-form]");
|
||||
this.errorBox = this.root.querySelector("[data-deepsearch-error]");
|
||||
this.live = this.root.querySelector("[data-deepsearch-live]");
|
||||
this.statusText = this.root.querySelector("[data-deepsearch-status]");
|
||||
this.countText = this.root.querySelector("[data-deepsearch-count]");
|
||||
this.bar = this.root.querySelector("[data-deepsearch-bar]");
|
||||
this.log = this.root.querySelector("[data-deepsearch-log]");
|
||||
this.done = this.root.querySelector("[data-deepsearch-done]");
|
||||
this.openLink = this.root.querySelector("[data-deepsearch-open]");
|
||||
this.pauseBtn = this.root.querySelector("[data-deepsearch-pause]");
|
||||
this.resumeBtn = this.root.querySelector("[data-deepsearch-resume]");
|
||||
this.cancelBtn = this.root.querySelector("[data-deepsearch-cancel]");
|
||||
this.runBtn = this.form.querySelector("[data-deepsearch-run]");
|
||||
this.socket = null;
|
||||
this.uid = null;
|
||||
this._bind();
|
||||
}
|
||||
|
||||
_bind() {
|
||||
this.form.addEventListener("submit", (event) => {
|
||||
event.preventDefault();
|
||||
this._start();
|
||||
});
|
||||
this.pauseBtn.addEventListener("click", () => this._control("pause"));
|
||||
this.resumeBtn.addEventListener("click", () => this._control("resume"));
|
||||
this.cancelBtn.addEventListener("click", () => this._control("cancel"));
|
||||
}
|
||||
|
||||
async _start() {
|
||||
this._setError("");
|
||||
const query = this.form.query.value.trim();
|
||||
if (!query) return;
|
||||
const params = {
|
||||
query,
|
||||
depth: this.form.depth ? this.form.depth.value : 2,
|
||||
max_pages: this.form.max_pages ? this.form.max_pages.value : 12,
|
||||
};
|
||||
this._resetLive();
|
||||
try {
|
||||
const data = await Http.send("/tools/deepsearch/run", params);
|
||||
this.uid = data.uid;
|
||||
this._watch(data.uid);
|
||||
} catch (error) {
|
||||
this._setError(error.message || "Could not start the research.");
|
||||
this.live.hidden = true;
|
||||
this.runBtn.disabled = false;
|
||||
}
|
||||
}
|
||||
|
||||
_resetLive() {
|
||||
if (this.socket) this.socket.close();
|
||||
this.done.hidden = true;
|
||||
this.live.hidden = false;
|
||||
this.log.innerHTML = "";
|
||||
this.bar.style.width = "0%";
|
||||
this.countText.textContent = "";
|
||||
this.statusText.textContent = "Starting…";
|
||||
this.runBtn.disabled = true;
|
||||
this.pauseBtn.hidden = false;
|
||||
this.resumeBtn.hidden = true;
|
||||
this.cancelBtn.hidden = false;
|
||||
}
|
||||
|
||||
async _control(action) {
|
||||
if (!this.uid) return;
|
||||
try {
|
||||
await Http.send(`/tools/deepsearch/${this.uid}/${action}`, {});
|
||||
} catch (error) {
|
||||
this._setError(error.message || "Control failed.");
|
||||
return;
|
||||
}
|
||||
if (action === "pause") {
|
||||
this.pauseBtn.hidden = true;
|
||||
this.resumeBtn.hidden = false;
|
||||
this.statusText.textContent = "Paused";
|
||||
} else if (action === "resume") {
|
||||
this.pauseBtn.hidden = false;
|
||||
this.resumeBtn.hidden = true;
|
||||
} else if (action === "cancel") {
|
||||
this.pauseBtn.hidden = true;
|
||||
this.resumeBtn.hidden = true;
|
||||
this.cancelBtn.hidden = true;
|
||||
this.statusText.textContent = "Cancelling…";
|
||||
}
|
||||
}
|
||||
|
||||
_watch(uid) {
|
||||
this.socket = new DeepsearchProgressSocket(uid, {
|
||||
onMessage: (frame) => this._frame(frame),
|
||||
onClose: () => {},
|
||||
});
|
||||
this.socket.connect();
|
||||
}
|
||||
|
||||
_frame(frame) {
|
||||
const type = frame.type;
|
||||
if (type === "stage") {
|
||||
this.statusText.textContent = frame.message || "Working…";
|
||||
this._appendLog(frame.message || frame.stage);
|
||||
} else if (type === "queries") {
|
||||
this._appendLog(`Planned ${frame.queries.length} queries`);
|
||||
} else if (type === "candidates") {
|
||||
this._appendLog(`Found ${frame.count} candidate sources`);
|
||||
} else if (type === "progress") {
|
||||
const total = frame.total || 1;
|
||||
const done = frame.done || 0;
|
||||
this.bar.style.width = `${Math.round((done / total) * 100)}%`;
|
||||
this.countText.textContent = `${done} / ${total} sources`;
|
||||
if (frame.message) this.statusText.textContent = frame.message;
|
||||
} else if (type === "page_loaded") {
|
||||
this.bar.style.width = `${Math.round((frame.done / frame.total) * 100)}%`;
|
||||
this.countText.textContent = `${frame.done} / ${frame.total} sources`;
|
||||
this._appendLog(`${frame.source} ${frame.title || frame.url}`);
|
||||
} else if (type === "agent") {
|
||||
this.statusText.textContent = frame.message || "Analysing";
|
||||
this._appendLog(`Agent: ${frame.agent}`);
|
||||
} else if (type === "report_ready") {
|
||||
this.statusText.textContent = "Compiling report…";
|
||||
} else if (type === "done") {
|
||||
this.bar.style.width = "100%";
|
||||
this.statusText.textContent = "Done";
|
||||
this._finish(frame.session_url || `/tools/deepsearch/${this.uid}/session`);
|
||||
} else if (type === "failed") {
|
||||
this.statusText.textContent = "Failed";
|
||||
this._setError(frame.message || frame.error || "The research failed.");
|
||||
this.runBtn.disabled = false;
|
||||
this.pauseBtn.hidden = true;
|
||||
this.cancelBtn.hidden = true;
|
||||
}
|
||||
}
|
||||
|
||||
_finish(sessionUrl) {
|
||||
this.runBtn.disabled = false;
|
||||
this.pauseBtn.hidden = true;
|
||||
this.resumeBtn.hidden = true;
|
||||
this.cancelBtn.hidden = true;
|
||||
this.done.hidden = false;
|
||||
this.openLink.href = sessionUrl;
|
||||
}
|
||||
|
||||
_appendLog(text) {
|
||||
if (!text) return;
|
||||
const item = document.createElement("li");
|
||||
item.textContent = text;
|
||||
this.log.prepend(item);
|
||||
}
|
||||
|
||||
_setError(message) {
|
||||
if (!this.errorBox) return;
|
||||
this.errorBox.textContent = message;
|
||||
this.errorBox.hidden = !message;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,216 @@
|
||||
// retoor <retoor@molodetz.nl>
|
||||
|
||||
import { Component } from "./Component.js";
|
||||
import { assetUrl } from "../assetVersion.js";
|
||||
|
||||
const CHAT_CSS_ID = "deepsearch-chat-css";
|
||||
const RETRY_CODE = 4013;
|
||||
const RETRY_MS = 200;
|
||||
const RECONNECT_MS = 1500;
|
||||
|
||||
export class AppDeepsearchChat extends Component {
|
||||
constructor() {
|
||||
super();
|
||||
this.ws = null;
|
||||
this._shouldRun = true;
|
||||
this._busy = false;
|
||||
}
|
||||
|
||||
_ensureCss() {
|
||||
if (document.getElementById(CHAT_CSS_ID)) return;
|
||||
const link = document.createElement("link");
|
||||
link.id = CHAT_CSS_ID;
|
||||
link.rel = "stylesheet";
|
||||
link.href = assetUrl("/static/css/deepsearch.css");
|
||||
document.head.appendChild(link);
|
||||
}
|
||||
|
||||
connectedCallback() {
|
||||
if (this._built) return;
|
||||
this._built = true;
|
||||
this._ensureCss();
|
||||
this.uid = this.attr("uid");
|
||||
this.chatPath = this.attr("chat-ws") || `/tools/deepsearch/${this.uid}/chat`;
|
||||
this._build();
|
||||
this._open();
|
||||
}
|
||||
|
||||
disconnectedCallback() {
|
||||
this._shouldRun = false;
|
||||
if (this.ws) this.ws.close();
|
||||
}
|
||||
|
||||
_build() {
|
||||
this.innerHTML = "";
|
||||
this.log = document.createElement("div");
|
||||
this.log.className = "ds-chat-log";
|
||||
this.log.setAttribute("role", "log");
|
||||
this.log.setAttribute("aria-live", "polite");
|
||||
|
||||
this.statusEl = document.createElement("div");
|
||||
this.statusEl.className = "ds-chat-status";
|
||||
this.statusEl.hidden = true;
|
||||
|
||||
this.form = document.createElement("form");
|
||||
this.form.className = "ds-chat-input";
|
||||
this.field = document.createElement("textarea");
|
||||
this.field.className = "ds-chat-field";
|
||||
this.field.rows = 1;
|
||||
this.field.placeholder = "Ask about the gathered evidence...";
|
||||
this.send = document.createElement("button");
|
||||
this.send.type = "submit";
|
||||
this.send.className = "ds-chat-send";
|
||||
this.send.textContent = "Ask";
|
||||
this.form.append(this.field, this.send);
|
||||
|
||||
this.append(this.log, this.statusEl, this.form);
|
||||
|
||||
this.form.addEventListener("submit", (event) => {
|
||||
event.preventDefault();
|
||||
this._submit(this.field.value);
|
||||
});
|
||||
this.field.addEventListener("keydown", (event) => {
|
||||
if (event.key === "Enter" && !event.shiftKey) {
|
||||
event.preventDefault();
|
||||
this._submit(this.field.value);
|
||||
}
|
||||
});
|
||||
this.field.addEventListener("input", () => this._autosize());
|
||||
}
|
||||
|
||||
_wsUrl() {
|
||||
const scheme = location.protocol === "https:" ? "wss://" : "ws://";
|
||||
return `${scheme}${location.host}${this.chatPath}`;
|
||||
}
|
||||
|
||||
_open() {
|
||||
this.ws = new WebSocket(this._wsUrl());
|
||||
this.ws.addEventListener("message", (event) => {
|
||||
let frame;
|
||||
try {
|
||||
frame = JSON.parse(event.data);
|
||||
} catch {
|
||||
return;
|
||||
}
|
||||
this._onMessage(frame);
|
||||
});
|
||||
this.ws.addEventListener("close", (event) => {
|
||||
if (event.code === RETRY_CODE && this._shouldRun) {
|
||||
window.setTimeout(() => this._open(), RETRY_MS);
|
||||
return;
|
||||
}
|
||||
if (this._shouldRun) {
|
||||
this._setStatus("Reconnecting...");
|
||||
window.setTimeout(() => this._open(), RECONNECT_MS);
|
||||
}
|
||||
});
|
||||
this.ws.addEventListener("error", () => this._setStatus("Connection error."));
|
||||
}
|
||||
|
||||
_onMessage(frame) {
|
||||
if (frame.type === "ready") {
|
||||
this._setStatus("");
|
||||
this._renderHistory(frame.history || []);
|
||||
} else if (frame.type === "status") {
|
||||
this._setStatus(frame.text);
|
||||
} else if (frame.type === "reply") {
|
||||
this._busy = false;
|
||||
this._setStatus("");
|
||||
this._agent(frame.text, frame.citations);
|
||||
} else if (frame.type === "error") {
|
||||
this._busy = false;
|
||||
this._setStatus("");
|
||||
this._errorLine(frame.text);
|
||||
}
|
||||
}
|
||||
|
||||
_submit(raw) {
|
||||
const text = (raw || "").trim();
|
||||
if (!text || this._busy) return;
|
||||
if (!this.ws || this.ws.readyState !== WebSocket.OPEN) {
|
||||
this._errorLine("Not connected.");
|
||||
return;
|
||||
}
|
||||
this._busy = true;
|
||||
this._userLine(text);
|
||||
this.field.value = "";
|
||||
this._autosize();
|
||||
this.ws.send(JSON.stringify({ type: "input", text }));
|
||||
this._setStatus("Searching the research collection");
|
||||
}
|
||||
|
||||
_renderHistory(messages) {
|
||||
this.log.innerHTML = "";
|
||||
messages.forEach((message) => {
|
||||
if (message.role === "user") {
|
||||
this._userLine(message.content);
|
||||
} else if (message.role === "assistant" && message.content) {
|
||||
this._agent(message.content, []);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
_userLine(text) {
|
||||
const row = document.createElement("div");
|
||||
row.className = "ds-chat-msg user";
|
||||
const bubble = document.createElement("div");
|
||||
bubble.className = "ds-chat-bubble";
|
||||
bubble.textContent = text;
|
||||
row.appendChild(bubble);
|
||||
this._append(row);
|
||||
}
|
||||
|
||||
_agent(text, citations) {
|
||||
const row = document.createElement("div");
|
||||
row.className = "ds-chat-msg agent";
|
||||
const bubble = document.createElement("div");
|
||||
bubble.className = "ds-chat-bubble";
|
||||
const content = document.createElement("dp-content");
|
||||
content.setAttribute("data-render", "");
|
||||
content.textContent = text;
|
||||
bubble.appendChild(content);
|
||||
if (citations && citations.length) {
|
||||
const list = document.createElement("div");
|
||||
list.className = "ds-chat-citations";
|
||||
citations.forEach((cite) => {
|
||||
const link = document.createElement("a");
|
||||
link.href = cite.url;
|
||||
link.target = "_blank";
|
||||
link.rel = "noopener nofollow";
|
||||
link.textContent = `[${cite.index}] ${cite.title}`;
|
||||
list.appendChild(link);
|
||||
});
|
||||
bubble.appendChild(list);
|
||||
}
|
||||
row.appendChild(bubble);
|
||||
this._append(row);
|
||||
}
|
||||
|
||||
_errorLine(text) {
|
||||
const row = document.createElement("div");
|
||||
row.className = "ds-chat-msg error";
|
||||
const bubble = document.createElement("div");
|
||||
bubble.className = "ds-chat-bubble";
|
||||
bubble.textContent = text;
|
||||
row.appendChild(bubble);
|
||||
this._append(row);
|
||||
}
|
||||
|
||||
_append(node) {
|
||||
const pinned = this.log.scrollHeight - this.log.scrollTop - this.log.clientHeight < 80;
|
||||
this.log.appendChild(node);
|
||||
if (pinned) this.log.scrollTop = this.log.scrollHeight;
|
||||
}
|
||||
|
||||
_autosize() {
|
||||
this.field.style.height = "auto";
|
||||
this.field.style.height = `${Math.min(this.field.scrollHeight, 160)}px`;
|
||||
}
|
||||
|
||||
_setStatus(text) {
|
||||
this.statusEl.textContent = text || "";
|
||||
this.statusEl.hidden = !text;
|
||||
}
|
||||
}
|
||||
|
||||
customElements.define("dp-deepsearch-chat", AppDeepsearchChat);
|
||||
@@ -67,6 +67,7 @@
|
||||
<button type="button" class="topnav-link topnav-tools-toggle" aria-haspopup="true" aria-expanded="false"><span class="icon">🧰</span> Tools <span class="topnav-caret">▾</span></button>
|
||||
<div class="dropdown-menu">
|
||||
<a href="/tools/seo" class="dropdown-item"><span class="icon">🔍</span> SEO Diagnostics</a>
|
||||
<a href="/tools/deepsearch" class="dropdown-item"><span class="icon">🧠</span> DeepSearch</a>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
@@ -126,6 +127,7 @@
|
||||
<div class="topnav-mobile-divider"></div>
|
||||
<div class="topnav-mobile-section-label">Tools</div>
|
||||
<a href="/tools/seo" class="topnav-mobile-link {% if 'tools/seo' in request.url.path %}active{% endif %}"><span class="icon">🔍</span> SEO Diagnostics</a>
|
||||
<a href="/tools/deepsearch" class="topnav-mobile-link {% if 'tools/deepsearch' in request.url.path %}active{% endif %}"><span class="icon">🧠</span> DeepSearch</a>
|
||||
{% if user %}
|
||||
<a href="/messages" class="topnav-mobile-link {% if 'messages' in request.url.path %}active{% endif %}" data-counter="messages"><span class="icon">✉️</span> Messages<span class="nav-badge nav-badge-inline" data-counter-badge {% if get_unread_messages(user["uid"]) == 0 %}hidden{% endif %}>{{ get_unread_messages(user["uid"]) }}</span></a>
|
||||
<a href="/notifications" class="topnav-mobile-link {% if 'notifications' in request.url.path %}active{% endif %}" data-counter="notifications"><span class="icon">🔔</span> Notifications<span class="nav-badge nav-badge-inline" data-counter-badge {% if get_unread_count(user["uid"]) == 0 %}hidden{% endif %}>{{ get_unread_count(user["uid"]) }}</span></a>
|
||||
|
||||
@@ -32,6 +32,8 @@ Chat requests are augmented (vision), forwarded, retried, priced, and ledgered.
|
||||
|
||||
**Upstream:** `gateway_upstream_url` (default DeepSeek chat-completions), `gateway_model` (`deepseek-v4-flash`), `gateway_force_model` (override the client-requested model, default on), `gateway_api_key` (auto-migrated from `DEEPSEEK_API_KEY`/`OPENROUTER_API_KEY`), `gateway_timeout` (default and minimum 300s), `gateway_instances` (max concurrent forwards per worker, default 4, 1 to 64; this is the connection pool plus semaphore).
|
||||
|
||||
**Prompt:** `gateway_system_preamble` (multiline text, default empty) - operator text prepended ahead of every chat request's system message (see "System-message composition" below).
|
||||
|
||||
**Vision:** `gateway_vision_enabled` (default on), `gateway_vision_url` (OpenRouter by default), `gateway_vision_model` (`google/gemma-3-12b-it`), `gateway_vision_key` (auto-migrated from `OPENROUTER_API_KEY`), `gateway_vision_cache_size` (LRU entries, default 256, 0 disables).
|
||||
|
||||
**Access:** `gateway_require_auth` (default on), `gateway_allow_admins`, `gateway_allow_users` (both default on), `gateway_access_key` (secret, write-only), `gateway_internal_key` (auto-generated on boot; clear and restart to rotate).
|
||||
@@ -50,6 +52,14 @@ If the upstream response carries a numeric `cost` field, that value is used verb
|
||||
|
||||
`retry_send()` retries on connection errors, timeouts, and upstream 5xx, up to `max_retries`, with a linear backoff of `backoff_ms * attempt`. The `CircuitBreaker` counts consecutive failures: at `threshold` it opens and new requests are rejected immediately with `503` and error type `circuit_open`; after `cooldown_seconds` it half-opens to test recovery and closes on the next success. Setting the threshold to 0 disables the breaker.
|
||||
|
||||
## System-message composition
|
||||
|
||||
Before a chat request is forwarded (and only for `chat/completions`, never embeddings or passthrough), the gateway composes a single system message in the fixed order: operator preamble, then a date line, then the client's original system content.
|
||||
|
||||
The operator preamble (`gateway_system_preamble`) is prepended on every chat call so all calls carry your preferences. If the client sent no system message at all, the composed content becomes a new leading system message; a blank or whitespace-only preamble is a no-op.
|
||||
|
||||
The current date is injected as `Current date: DD/MM/YYYY` (EU format) only when the composed text contains no date in any common format already (ISO `YYYY-MM-DD`, `D/M/YYYY`, `DD/MM/YYYY`, `MM/DD/YYYY`, `DD-MM-YYYY`, `DD.MM.YYYY`, and textual months such as `14 June 2026`, `June 14, 2026`, `14 Jun 2026`). The date is intentionally date-only with no time: keeping it stable for the whole day preserves upstream prompt and token caching.
|
||||
|
||||
## Vision augmentation
|
||||
|
||||
Before forwarding a chat request, image blocks are described by the vision model and replaced with a text description, so a text-only upstream still answers questions about images. Descriptions are cached in an LRU keyed by image hash; vision calls are themselves ledgered with `backend="vision"`. If vision is disabled or the key is missing, images become a placeholder note rather than failing the request.
|
||||
|
||||
@@ -0,0 +1,66 @@
|
||||
<div class="docs-content" data-render>
|
||||
# DeepSearch
|
||||
|
||||
DeepSearch is a multi-agent deep web researcher. Given a single research question it plans a set of
|
||||
diverse web search queries, crawls and reads the most relevant sources, indexes everything into a
|
||||
private vector collection for that run, then runs a chain of agents (summarizer, critic, linker) to
|
||||
produce a grounded, cited report with a confidence score, source diversity and an explicit list of
|
||||
open gaps. Progress streams live while it works, and afterwards you can chat with the gathered
|
||||
evidence.
|
||||
|
||||
Open it from the **Tools** menu, or go straight to `/tools/deepsearch`. It is public: you do not
|
||||
need an account.
|
||||
|
||||
## Running a research job
|
||||
|
||||
1. Type a focused research question.
|
||||
2. Set the **depth** (1-4) and the maximum number of **pages** to crawl (up to 30).
|
||||
3. Press **Research**. Progress appears immediately: query planning, web search, crawling each
|
||||
source, indexing, then the analysis agents.
|
||||
4. You can **pause**, **resume** or **cancel** a run at any time.
|
||||
5. When it finishes, open the report to read the summary, findings, gaps and sources, and to chat
|
||||
with the research.
|
||||
|
||||
You can run one job at a time. Targets that resolve to private or local addresses are refused, and
|
||||
every fetched URL (including redirects) is checked.
|
||||
|
||||
## How it works
|
||||
|
||||
- **Query planning** expands your question into several complementary searches.
|
||||
- **Crawling** fetches each candidate first with a plain HTTP client, falling back to a headless
|
||||
browser for JavaScript-heavy pages. Identical content is de-duplicated, and a cross-session URL
|
||||
cache avoids re-fetching pages seen by earlier runs.
|
||||
- **Indexing** splits each page into overlapping chunks, embeds them through the AI gateway (with a
|
||||
local embedding fallback when the gateway is unavailable), and stores them in a per-session
|
||||
ChromaDB collection.
|
||||
- **Analysis** runs the summarizer, critic and linker agents to synthesise findings, surface gaps,
|
||||
and score overall confidence. The score combines confidence, source diversity and coverage.
|
||||
|
||||
## Chatting with the research
|
||||
|
||||
Every finished session has a chat pane. Answers are grounded **only** in the sources captured during
|
||||
that run, using hybrid retrieval (vector similarity plus keyword/BM25 ranking) over the session
|
||||
collection, and every claim is cited back to a source.
|
||||
|
||||
## Exporting
|
||||
|
||||
A finished report can be downloaded as **Markdown**, **JSON** or **PDF** from the report page.
|
||||
|
||||
## Ask Devii
|
||||
|
||||
You can also run research in plain language through the Devii assistant:
|
||||
|
||||
> Run a deep search on the history of the transistor and summarise the findings.
|
||||
|
||||
Devii queues the job, polls it, and reports the score, confidence and a link to the report.
|
||||
|
||||
## Programmatic access
|
||||
|
||||
The same research is available over the API: `POST /tools/deepsearch/run` to queue,
|
||||
`GET /tools/deepsearch/{uid}` to poll, and `GET /tools/deepsearch/{uid}/session` for the full report
|
||||
(HTML or JSON). See the [Tools](/docs/tools.html) API group for request and response shapes.
|
||||
</div>
|
||||
|
||||
<div class="devii-doc-cta">
|
||||
<a href="/tools/deepsearch" class="sidebar-link">Open DeepSearch</a>
|
||||
</div>
|
||||
@@ -0,0 +1,98 @@
|
||||
{% extends "base.html" %}
|
||||
|
||||
{% block extra_head %}
|
||||
<link rel="stylesheet" href="{{ static_url('/static/css/sidebar.css') }}">
|
||||
<link rel="stylesheet" href="{{ static_url('/static/css/deepsearch.css') }}">
|
||||
{% endblock %}
|
||||
|
||||
{% block content %}
|
||||
<div class="ds-layout" data-deepsearch-tool>
|
||||
<aside class="sidebar-card">
|
||||
<div class="sidebar-heading">DeepSearch</div>
|
||||
<div class="sidebar-nav">
|
||||
<a href="/tools" class="sidebar-link">
|
||||
<span class="icon">🧰</span>
|
||||
All tools
|
||||
</a>
|
||||
<a href="/tools/deepsearch" class="sidebar-link active">
|
||||
<span class="icon">🧠</span>
|
||||
Research
|
||||
</a>
|
||||
</div>
|
||||
|
||||
<div class="sidebar-section">
|
||||
<div class="sidebar-heading">How it works</div>
|
||||
<ul class="ds-side-list">
|
||||
<li>Plans diverse web search queries</li>
|
||||
<li>Crawls and indexes sources</li>
|
||||
<li>Multi-agent synthesis and critique</li>
|
||||
<li>Confidence and source diversity</li>
|
||||
<li>Grounded chat over the results</li>
|
||||
</ul>
|
||||
</div>
|
||||
|
||||
<div class="sidebar-section">
|
||||
<div class="sidebar-heading">Tips</div>
|
||||
<ul class="ds-side-list">
|
||||
<li>Ask one focused research question.</li>
|
||||
<li>Deeper runs crawl more sources.</li>
|
||||
<li>Only one run at a time per visitor.</li>
|
||||
<li>Pause, resume or cancel any time.</li>
|
||||
</ul>
|
||||
</div>
|
||||
</aside>
|
||||
|
||||
<div class="ds-main">
|
||||
<header class="ds-header">
|
||||
<h1><span class="icon">🧠</span> DeepSearch</h1>
|
||||
<p>A multi-agent deep web researcher. It plans queries, crawls and indexes sources,
|
||||
then synthesises a cited report you can chat with. Progress streams live.</p>
|
||||
</header>
|
||||
|
||||
<section class="ds-form-card">
|
||||
<form class="ds-form" data-deepsearch-form autocomplete="off">
|
||||
<textarea name="query" class="ds-input" rows="2"
|
||||
placeholder="What do you want to research?" required></textarea>
|
||||
<div class="ds-form-row">
|
||||
<label class="ds-field">
|
||||
Depth
|
||||
<input type="number" name="depth" class="ds-input-num" value="2" min="1" max="4">
|
||||
</label>
|
||||
<label class="ds-field">
|
||||
Pages
|
||||
<input type="number" name="max_pages" class="ds-input-num" value="12" min="1" max="30">
|
||||
</label>
|
||||
<button type="submit" class="btn btn-primary ds-run-btn" data-deepsearch-run>Research</button>
|
||||
</div>
|
||||
<p class="ds-form-error" data-deepsearch-error hidden></p>
|
||||
</form>
|
||||
</section>
|
||||
|
||||
<section class="ds-live" data-deepsearch-live hidden>
|
||||
<div class="ds-live-head">
|
||||
<span class="ds-live-status" data-deepsearch-status>Starting…</span>
|
||||
<span class="ds-live-count" data-deepsearch-count></span>
|
||||
<span class="ds-live-controls">
|
||||
<button type="button" class="btn ds-ctl" data-deepsearch-pause hidden>Pause</button>
|
||||
<button type="button" class="btn ds-ctl" data-deepsearch-resume hidden>Resume</button>
|
||||
<button type="button" class="btn ds-ctl" data-deepsearch-cancel hidden>Cancel</button>
|
||||
</span>
|
||||
</div>
|
||||
<div class="ds-progress"><div class="ds-progress-bar" data-deepsearch-bar></div></div>
|
||||
<ul class="ds-timeline" data-deepsearch-log></ul>
|
||||
</section>
|
||||
|
||||
<section class="ds-done" data-deepsearch-done hidden>
|
||||
<p>Research complete.</p>
|
||||
<a class="btn btn-primary" data-deepsearch-open href="#">Open report and chat</a>
|
||||
</section>
|
||||
</div>
|
||||
</div>
|
||||
{% endblock %}
|
||||
|
||||
{% block extra_js %}
|
||||
<script type="module">
|
||||
import { DeepsearchTool } from "{{ static_url('/static/js/DeepsearchTool.js') }}";
|
||||
window.app.deepsearchTool = new DeepsearchTool();
|
||||
</script>
|
||||
{% endblock %}
|
||||
@@ -0,0 +1,94 @@
|
||||
{% extends "base.html" %}
|
||||
|
||||
{% block extra_head %}
|
||||
<link rel="stylesheet" href="{{ static_url('/static/css/deepsearch.css') }}">
|
||||
{% endblock %}
|
||||
|
||||
{% block content %}
|
||||
<div class="ds-session" data-deepsearch-session data-uid="{{ uid }}"
|
||||
{% if chat_ws_url %}data-chat-ws="{{ chat_ws_url }}"{% endif %}>
|
||||
<div class="ds-session-main">
|
||||
<header class="ds-session-head">
|
||||
<h1>{{ query }}</h1>
|
||||
<div class="ds-metrics">
|
||||
<span class="ds-metric"><strong>{{ score }}</strong> score</span>
|
||||
<span class="ds-metric"><strong>{{ confidence }}</strong> confidence</span>
|
||||
<span class="ds-metric"><strong>{{ source_diversity }}</strong> diversity</span>
|
||||
<span class="ds-metric"><strong>{{ page_count }}</strong> sources</span>
|
||||
<span class="ds-metric"><strong>{{ chunk_count }}</strong> chunks</span>
|
||||
</div>
|
||||
{% if export_md_url %}
|
||||
<div class="ds-exports">
|
||||
<a class="btn ds-export" href="{{ export_md_url }}">Markdown</a>
|
||||
<a class="btn ds-export" href="{{ export_json_url }}">JSON</a>
|
||||
<a class="btn ds-export" href="{{ export_pdf_url }}">PDF</a>
|
||||
</div>
|
||||
{% endif %}
|
||||
</header>
|
||||
|
||||
{% if summary %}
|
||||
<section class="ds-card">
|
||||
<h2>Summary</h2>
|
||||
<div class="ds-summary rendered-content" data-render>{{ summary }}</div>
|
||||
</section>
|
||||
{% endif %}
|
||||
|
||||
{% if findings %}
|
||||
<section class="ds-card">
|
||||
<h2>Findings</h2>
|
||||
<div class="ds-findings">
|
||||
{% for finding in findings %}
|
||||
<details class="ds-finding" open>
|
||||
<summary>
|
||||
<span class="ds-finding-title">{{ finding.title }}</span>
|
||||
<span class="ds-finding-confidence">{{ finding.confidence }}</span>
|
||||
</summary>
|
||||
<div class="ds-finding-detail">{{ finding.detail }}</div>
|
||||
</details>
|
||||
{% endfor %}
|
||||
</div>
|
||||
</section>
|
||||
{% endif %}
|
||||
|
||||
{% if gaps %}
|
||||
<section class="ds-card">
|
||||
<h2>Open gaps</h2>
|
||||
<ul class="ds-gaps">
|
||||
{% for gap in gaps %}
|
||||
<li>{{ gap }}</li>
|
||||
{% endfor %}
|
||||
</ul>
|
||||
</section>
|
||||
{% endif %}
|
||||
|
||||
{% if sources %}
|
||||
<section class="ds-card">
|
||||
<h2>Sources</h2>
|
||||
<ol class="ds-sources">
|
||||
{% for source in sources %}
|
||||
<li>
|
||||
<a href="{{ source.url }}" target="_blank" rel="noopener nofollow">{{ source.title or source.url }}</a>
|
||||
<span class="ds-source-tag">{{ source.source }}</span>
|
||||
</li>
|
||||
{% endfor %}
|
||||
</ol>
|
||||
</section>
|
||||
{% endif %}
|
||||
</div>
|
||||
|
||||
<aside class="ds-chat-pane">
|
||||
<div class="ds-chat-head">Ask the research</div>
|
||||
{% if chat_ws_url %}
|
||||
<dp-deepsearch-chat data-uid="{{ uid }}" data-chat-ws="{{ chat_ws_url }}"></dp-deepsearch-chat>
|
||||
{% else %}
|
||||
<p class="ds-chat-pending">Chat opens once the research finishes.</p>
|
||||
{% endif %}
|
||||
</aside>
|
||||
</div>
|
||||
{% endblock %}
|
||||
|
||||
{% block extra_js %}
|
||||
<script type="module">
|
||||
import "{{ static_url('/static/js/components/AppDeepsearchChat.js') }}";
|
||||
</script>
|
||||
{% endblock %}
|
||||
Reference in New Issue
Block a user