feat: add deepsearch research system with CLI prune/clear and database schema

Implement a multi-agent deep web research subsystem including CLI commands for pruning expired jobs and clearing all artifacts, database tables for sessions/messages/URL cache with indexes, config paths for chroma storage, and internal embed URL for vector operations.

devplacepy/services/audit/categories.py

@@ -27,6 +27,7 @@ CATEGORY_BY_PREFIX: dict[str, str] = {
     "container": "container",
     "proxy": "ingress",
     "seo": "tools",
+    "deepsearch": "tools",
     "ai": "ai",
     "devii": "devii",
     "cli": "cli",

devplacepy/services/deepsearch/__init__.py

@@ -0,0 +1 @@
+# retoor <retoor@molodetz.nl>

devplacepy/services/deepsearch/chat.py

@@ -0,0 +1,107 @@
+# retoor <retoor@molodetz.nl>
+
+from __future__ import annotations
+
+import logging
+from dataclasses import dataclass, field
+
+from .embeddings import embed_texts, local_embed
+from .llm import complete_chat
+from .store import Chunk, VectorStore
+
+logger = logging.getLogger(__name__)
+
+CHAT_TOP_K = 8
+MAX_CONTEXT_CHARS = 9000
+CHAT_MAX_TOKENS = 900
+
+SYSTEM_PROMPT = (
+    "You are the DeepSearch research assistant. Answer the user's question using ONLY "
+    "the numbered SOURCES below, which were gathered during a web research session. "
+    "Never use outside knowledge or guess. If the sources do not contain the answer, "
+    "say so plainly. Cite every claim inline with the bracket marker of the source it "
+    "comes from, like [1] or [2]. Keep the answer focused and well structured in "
+    "markdown."
+)
+
+
+@dataclass
+class ChatAnswer:
+    text: str
+    citations: list[dict] = field(default_factory=list)
+
+
+def _build_context(chunks: list[Chunk]) -> tuple[str, list[dict]]:
+    blocks: list[str] = []
+    citations: list[dict] = []
+    used = 0
+    for index, chunk in enumerate(chunks, start=1):
+        snippet = chunk.text.strip()
+        if not snippet:
+            continue
+        header = f"[{index}] {chunk.title or chunk.url} ({chunk.url})"
+        block = f"{header}\n{snippet}"
+        if used + len(block) > MAX_CONTEXT_CHARS and blocks:
+            break
+        used += len(block)
+        blocks.append(block)
+        citations.append(
+            {
+                "index": index,
+                "url": chunk.url,
+                "title": chunk.title or chunk.url,
+                "score": round(chunk.score, 4),
+            }
+        )
+    return "\n\n".join(blocks), citations
+
+
+class DeepsearchChat:
+    def __init__(self, collection_name: str, api_key: str) -> None:
+        self.store = VectorStore(collection_name)
+        self.api_key = api_key
+
+    async def _embed_query(self, question: str) -> list[float]:
+        result = await embed_texts([question], self.api_key)
+        if not result.vectors:
+            result = local_embed([question])
+        return result.vectors[0]
+
+    async def retrieve(self, question: str) -> list[Chunk]:
+        query_vector = await self._embed_query(question)
+        return self.store.hybrid_search(question, query_vector, top_k=CHAT_TOP_K)
+
+    async def answer(self, question: str, history: list[dict] | None = None) -> ChatAnswer:
+        chunks = await self.retrieve(question)
+        if not chunks:
+            return ChatAnswer(
+                text=(
+                    "The research session did not capture anything relevant to that "
+                    "question. Try rephrasing or running a deeper search."
+                ),
+                citations=[],
+            )
+        context, citations = _build_context(chunks)
+        messages = [{"role": "system", "content": SYSTEM_PROMPT}]
+        for turn in (history or [])[-6:]:
+            role = turn.get("role")
+            content = turn.get("content")
+            if role in ("user", "assistant") and content:
+                messages.append({"role": role, "content": content})
+        messages.append(
+            {
+                "role": "user",
+                "content": f"SOURCES:\n{context}\n\nQUESTION: {question}",
+            }
+        )
+        try:
+            text = await complete_chat(
+                messages, self.api_key, max_tokens=CHAT_MAX_TOKENS
+            )
+        except Exception as exc:
+            logger.warning("deepsearch chat synthesis failed: %s", exc)
+            text = (
+                "I could not reach the language model to synthesise an answer, but the "
+                "most relevant sources are listed below."
+            )
+        return ChatAnswer(text=text, citations=citations)

devplacepy/services/deepsearch/embeddings.py

@@ -0,0 +1,71 @@
+# retoor <retoor@molodetz.nl>
+
+from __future__ import annotations
+
+import hashlib
+import logging
+import math
+import re
+from dataclasses import dataclass
+
+import httpx
+
+from devplacepy.config import INTERNAL_EMBED_MODEL, INTERNAL_EMBED_URL
+
+logger = logging.getLogger(__name__)
+
+EMBED_TIMEOUT_SECONDS = 60.0
+LOCAL_EMBED_DIMS = 256
+TOKEN_PATTERN = re.compile(r"[a-z0-9]+")
+
+
+@dataclass
+class EmbedResult:
+    vectors: list[list[float]]
+    backend: str
+
+
+def _local_vector(text: str) -> list[float]:
+    bucket = [0.0] * LOCAL_EMBED_DIMS
+    tokens = TOKEN_PATTERN.findall((text or "").lower())
+    if not tokens:
+        return bucket
+    for token in tokens:
+        digest = hashlib.sha1(token.encode("utf-8")).digest()
+        index = int.from_bytes(digest[:4], "big") % LOCAL_EMBED_DIMS
+        sign = 1.0 if digest[4] % 2 == 0 else -1.0
+        bucket[index] += sign
+    norm = math.sqrt(sum(value * value for value in bucket))
+    if norm == 0.0:
+        return bucket
+    return [value / norm for value in bucket]
+
+
+def local_embed(texts: list[str]) -> EmbedResult:
+    return EmbedResult(vectors=[_local_vector(text) for text in texts], backend="local")
+
+
+async def embed_texts(
+    texts: list[str], api_key: str, *, gateway_url: str = INTERNAL_EMBED_URL
+) -> EmbedResult:
+    if not texts:
+        return EmbedResult(vectors=[], backend="empty")
+    headers = {
+        "Authorization": f"Bearer {api_key}",
+        "Content-Type": "application/json",
+    }
+    payload = {"model": INTERNAL_EMBED_MODEL, "input": texts}
+    try:
+        async with httpx.AsyncClient(timeout=EMBED_TIMEOUT_SECONDS) as client:
+            response = await client.post(gateway_url, json=payload, headers=headers)
+        if response.status_code >= 400:
+            raise RuntimeError(f"embed gateway returned {response.status_code}")
+        data = response.json()
+        rows = data.get("data") or []
+        vectors = [row.get("embedding") or [] for row in rows]
+        if len(vectors) != len(texts) or any(not vector for vector in vectors):
+            raise RuntimeError("embed gateway returned an incomplete response")
+        return EmbedResult(vectors=vectors, backend="gateway")
+    except Exception as exc:
+        logger.warning("deepsearch embedding gateway failed, using local: %s", exc)
+        return local_embed(texts)

devplacepy/services/deepsearch/export.py

@@ -0,0 +1,132 @@
+# retoor <retoor@molodetz.nl>
+
+from __future__ import annotations
+
+import html
+import json
+import logging
+from datetime import datetime, timezone
+
+logger = logging.getLogger(__name__)
+
+
+def _findings(report: dict) -> list[dict]:
+    return report.get("findings") or []
+
+
+def _sources(report: dict) -> list[dict]:
+    return report.get("sources") or []
+
+
+def _gaps(report: dict) -> list[str]:
+    return report.get("gaps") or []
+
+
+def to_markdown(report: dict) -> str:
+    query = report.get("query", "")
+    lines: list[str] = [f"# DeepSearch report: {query}", ""]
+    lines.append(
+        f"- Score: {report.get('score', 0)}  "
+        f"Confidence: {report.get('confidence', 0)}  "
+        f"Source diversity: {report.get('source_diversity', 0)}"
+    )
+    lines.append(
+        f"- Pages crawled: {report.get('page_count', 0)}  "
+        f"Chunks indexed: {report.get('chunk_count', 0)}"
+    )
+    generated = report.get("generated_at") or datetime.now(timezone.utc).isoformat()
+    lines.append(f"- Generated: {generated}")
+    lines.append("")
+    summary = report.get("summary", "")
+    if summary:
+        lines.extend(["## Summary", "", summary, ""])
+    findings = _findings(report)
+    if findings:
+        lines.append("## Findings")
+        lines.append("")
+        for finding in findings:
+            title = finding.get("title", "")
+            detail = finding.get("detail", "")
+            confidence = finding.get("confidence", 0)
+            lines.append(f"### {title}")
+            lines.append("")
+            lines.append(detail)
+            citations = finding.get("citations") or []
+            if citations:
+                lines.append("")
+                lines.append("Sources: " + ", ".join(str(c) for c in citations))
+            lines.append(f"\nConfidence: {confidence}")
+            lines.append("")
+    gaps = _gaps(report)
+    if gaps:
+        lines.append("## Open gaps")
+        lines.append("")
+        for gap in gaps:
+            lines.append(f"- {gap}")
+        lines.append("")
+    sources = _sources(report)
+    if sources:
+        lines.append("## Sources")
+        lines.append("")
+        for index, source in enumerate(sources, start=1):
+            title = source.get("title") or source.get("url", "")
+            url = source.get("url", "")
+            lines.append(f"{index}. [{title}]({url})")
+        lines.append("")
+    return "\n".join(lines)
+
+
+def to_json(report: dict) -> str:
+    return json.dumps(report, ensure_ascii=False, indent=2)
+
+
+def _html_document(report: dict) -> str:
+    query = html.escape(report.get("query", ""))
+    parts: list[str] = [
+        "<html><head><meta charset='utf-8'><style>",
+        "body{font-family:Arial,Helvetica,sans-serif;color:#1b2330;margin:40px;}",
+        "h1{font-size:22px;}h2{font-size:17px;margin-top:24px;}h3{font-size:14px;}",
+        ".meta{color:#566;font-size:12px;}a{color:#2d6cdf;}",
+        "</style></head><body>",
+        f"<h1>DeepSearch report: {query}</h1>",
+        "<p class='meta'>"
+        f"Score {report.get('score', 0)} | Confidence {report.get('confidence', 0)} | "
+        f"Diversity {report.get('source_diversity', 0)} | "
+        f"Pages {report.get('page_count', 0)} | Chunks {report.get('chunk_count', 0)}"
+        "</p>",
+    ]
+    summary = report.get("summary", "")
+    if summary:
+        parts.append("<h2>Summary</h2>")
+        parts.append(f"<p>{html.escape(summary)}</p>")
+    findings = _findings(report)
+    if findings:
+        parts.append("<h2>Findings</h2>")
+        for finding in findings:
+            parts.append(f"<h3>{html.escape(finding.get('title', ''))}</h3>")
+            parts.append(f"<p>{html.escape(finding.get('detail', ''))}</p>")
+            parts.append(
+                f"<p class='meta'>Confidence {finding.get('confidence', 0)}</p>"
+            )
+    gaps = _gaps(report)
+    if gaps:
+        parts.append("<h2>Open gaps</h2><ul>")
+        for gap in gaps:
+            parts.append(f"<li>{html.escape(gap)}</li>")
+        parts.append("</ul>")
+    sources = _sources(report)
+    if sources:
+        parts.append("<h2>Sources</h2><ol>")
+        for source in sources:
+            url = html.escape(source.get("url", ""))
+            title = html.escape(source.get("title") or source.get("url", ""))
+            parts.append(f"<li><a href='{url}'>{title}</a></li>")
+        parts.append("</ol>")
+    parts.append("</body></html>")
+    return "".join(parts)
+
+
+def to_pdf(report: dict) -> bytes:
+    from weasyprint import HTML
+
+    return HTML(string=_html_document(report)).write_pdf()

devplacepy/services/deepsearch/llm.py

@@ -0,0 +1,44 @@
+# retoor <retoor@molodetz.nl>
+
+from __future__ import annotations
+
+import logging
+
+import httpx
+
+from devplacepy.config import INTERNAL_GATEWAY_URL, INTERNAL_MODEL
+
+logger = logging.getLogger(__name__)
+
+CHAT_TIMEOUT_SECONDS = 120.0
+DEFAULT_MAX_TOKENS = 1200
+
+
+async def complete_chat(
+    messages: list[dict],
+    api_key: str,
+    *,
+    gateway_url: str = INTERNAL_GATEWAY_URL,
+    model: str = INTERNAL_MODEL,
+    max_tokens: int = DEFAULT_MAX_TOKENS,
+    temperature: float = 0.2,
+) -> str:
+    payload = {
+        "model": model,
+        "messages": messages,
+        "max_tokens": max_tokens,
+        "temperature": temperature,
+    }
+    headers = {
+        "Authorization": f"Bearer {api_key}",
+        "Content-Type": "application/json",
+    }
+    async with httpx.AsyncClient(timeout=CHAT_TIMEOUT_SECONDS) as client:
+        response = await client.post(gateway_url, json=payload, headers=headers)
+    if response.status_code >= 400:
+        raise RuntimeError(f"chat gateway returned {response.status_code}")
+    data = response.json()
+    choices = data.get("choices") or []
+    if not choices:
+        raise RuntimeError("chat gateway returned no choices")
+    return (choices[0].get("message", {}).get("content") or "").strip()

devplacepy/services/deepsearch/store.py

@@ -0,0 +1,209 @@
+# retoor <retoor@molodetz.nl>
+
+from __future__ import annotations
+
+import logging
+import math
+import re
+from collections import Counter
+from dataclasses import dataclass, field
+
+from devplacepy.config import DEEPSEARCH_CHROMA_DIR
+
+logger = logging.getLogger(__name__)
+
+TOKEN_PATTERN = re.compile(r"[a-z0-9]+")
+BM25_K1 = 1.5
+BM25_B = 0.75
+HYBRID_VECTOR_WEIGHT = 0.6
+HYBRID_KEYWORD_WEIGHT = 0.4
+DEFAULT_TOP_K = 8
+CANDIDATE_MULTIPLIER = 4
+
+
+@dataclass
+class Chunk:
+    uid: str
+    text: str
+    url: str
+    title: str
+    depth: int = 0
+    source: str = ""
+    position: int = 0
+    score: float = 0.0
+    metadata: dict = field(default_factory=dict)
+
+
+def _tokenize(text: str) -> list[str]:
+    return TOKEN_PATTERN.findall((text or "").lower())
+
+
+class VectorStore:
+    def __init__(self, collection_name: str) -> None:
+        self.collection_name = collection_name
+        self._client = None
+        self._collection = None
+
+    def _ensure(self):
+        if self._collection is not None:
+            return self._collection
+        import chromadb
+
+        DEEPSEARCH_CHROMA_DIR.mkdir(parents=True, exist_ok=True)
+        self._client = chromadb.PersistentClient(path=str(DEEPSEARCH_CHROMA_DIR))
+        self._collection = self._client.get_or_create_collection(
+            name=self.collection_name, metadata={"hnsw:space": "cosine"}
+        )
+        return self._collection
+
+    def add(self, chunks: list[Chunk], vectors: list[list[float]]) -> None:
+        if not chunks:
+            return
+        collection = self._ensure()
+        collection.add(
+            ids=[chunk.uid for chunk in chunks],
+            embeddings=vectors,
+            documents=[chunk.text for chunk in chunks],
+            metadatas=[
+                {
+                    "url": chunk.url,
+                    "title": chunk.title,
+                    "depth": chunk.depth,
+                    "source": chunk.source,
+                    "position": chunk.position,
+                }
+                for chunk in chunks
+            ],
+        )
+
+    def all_chunks(self) -> list[Chunk]:
+        collection = self._ensure()
+        data = collection.get(include=["documents", "metadatas"])
+        chunks: list[Chunk] = []
+        ids = data.get("ids") or []
+        documents = data.get("documents") or []
+        metadatas = data.get("metadatas") or []
+        for index, uid in enumerate(ids):
+            meta = metadatas[index] if index < len(metadatas) else {}
+            chunks.append(
+                Chunk(
+                    uid=uid,
+                    text=documents[index] if index < len(documents) else "",
+                    url=str(meta.get("url", "")),
+                    title=str(meta.get("title", "")),
+                    depth=int(meta.get("depth", 0) or 0),
+                    source=str(meta.get("source", "")),
+                    position=int(meta.get("position", 0) or 0),
+                    metadata=dict(meta),
+                )
+            )
+        return chunks
+
+    def count(self) -> int:
+        try:
+            return self._ensure().count()
+        except Exception:
+            return 0
+
+    def vector_search(
+        self, query_vector: list[float], top_k: int, where: dict | None = None
+    ) -> list[Chunk]:
+        collection = self._ensure()
+        result = collection.query(
+            query_embeddings=[query_vector],
+            n_results=top_k,
+            where=where or None,
+            include=["documents", "metadatas", "distances"],
+        )
+        ids = (result.get("ids") or [[]])[0]
+        documents = (result.get("documents") or [[]])[0]
+        metadatas = (result.get("metadatas") or [[]])[0]
+        distances = (result.get("distances") or [[]])[0]
+        chunks: list[Chunk] = []
+        for index, uid in enumerate(ids):
+            meta = metadatas[index] if index < len(metadatas) else {}
+            distance = distances[index] if index < len(distances) else 1.0
+            chunks.append(
+                Chunk(
+                    uid=uid,
+                    text=documents[index] if index < len(documents) else "",
+                    url=str(meta.get("url", "")),
+                    title=str(meta.get("title", "")),
+                    depth=int(meta.get("depth", 0) or 0),
+                    source=str(meta.get("source", "")),
+                    position=int(meta.get("position", 0) or 0),
+                    score=1.0 - float(distance),
+                    metadata=dict(meta),
+                )
+            )
+        return chunks
+
+    def keyword_scores(self, query: str, chunks: list[Chunk]) -> dict[str, float]:
+        terms = _tokenize(query)
+        if not terms or not chunks:
+            return {}
+        docs = [_tokenize(chunk.text) for chunk in chunks]
+        lengths = [len(doc) for doc in docs]
+        avg_len = (sum(lengths) / len(lengths)) if lengths else 0.0
+        doc_freq: Counter = Counter()
+        for doc in docs:
+            for term in set(doc):
+                if term in terms:
+                    doc_freq[term] += 1
+        total_docs = len(docs)
+        scores: dict[str, float] = {}
+        for index, chunk in enumerate(chunks):
+            counts = Counter(docs[index])
+            length = lengths[index] or 1
+            score = 0.0
+            for term in terms:
+                freq = counts.get(term, 0)
+                if freq == 0:
+                    continue
+                idf = math.log(
+                    1 + (total_docs - doc_freq[term] + 0.5) / (doc_freq[term] + 0.5)
+                )
+                denom = freq + BM25_K1 * (
+                    1 - BM25_B + BM25_B * (length / (avg_len or 1))
+                )
+                score += idf * (freq * (BM25_K1 + 1)) / (denom or 1)
+            scores[chunk.uid] = score
+        return scores
+
+    def hybrid_search(
+        self,
+        query: str,
+        query_vector: list[float],
+        top_k: int = DEFAULT_TOP_K,
+        where: dict | None = None,
+    ) -> list[Chunk]:
+        candidates = self.vector_search(
+            query_vector, top_k * CANDIDATE_MULTIPLIER, where
+        )
+        if not candidates:
+            return []
+        keyword = self.keyword_scores(query, candidates)
+        vec_max = max((chunk.score for chunk in candidates), default=0.0) or 1.0
+        kw_max = max(keyword.values(), default=0.0) or 1.0
+        for chunk in candidates:
+            vec_norm = max(0.0, chunk.score) / vec_max
+            kw_norm = keyword.get(chunk.uid, 0.0) / kw_max
+            chunk.score = (
+                HYBRID_VECTOR_WEIGHT * vec_norm + HYBRID_KEYWORD_WEIGHT * kw_norm
+            )
+        candidates.sort(key=lambda chunk: chunk.score, reverse=True)
+        return candidates[:top_k]
+
+    def drop(self) -> None:
+        try:
+            import chromadb
+
+            DEEPSEARCH_CHROMA_DIR.mkdir(parents=True, exist_ok=True)
+            client = self._client or chromadb.PersistentClient(
+                path=str(DEEPSEARCH_CHROMA_DIR)
+            )
+            client.delete_collection(self.collection_name)
+        except Exception as exc:
+            logger.info("deepsearch collection drop skipped for %s: %s", self.collection_name, exc)
+        finally:
+            self._collection = None

devplacepy/services/devii/actions/catalog.py

@@ -595,6 +595,48 @@ ACTIONS: tuple[Action, ...] = (
         params=(path("uid", "SEO job uid returned by seo_diagnostics."),),
         requires_auth=False,
     ),
+    Action(
+        name="deepsearch",
+        method="POST",
+        path="/tools/deepsearch/run",
+        summary="Start a deep multi-agent web research job",
+        description=(
+            "Queues a background DeepSearch job and returns {uid, status_url}. Poll the "
+            "status_url with deepsearch_status until status is 'done', then share the "
+            "score, confidence and session_url. depth (1-4) and max_pages (1-30) control "
+            "how widely it crawls."
+        ),
+        params=(
+            body("query", "The research question to investigate.", required=True),
+            body("depth", "Research depth 1-4 (default 2)."),
+            body("max_pages", "Maximum sources to crawl 1-30 (default 12)."),
+        ),
+        requires_auth=False,
+    ),
+    Action(
+        name="deepsearch_status",
+        method="GET",
+        path="/tools/deepsearch/{uid}",
+        summary="Check a DeepSearch job and obtain its metrics once finished",
+        description=(
+            "Returns the research status. When status is 'done', score, confidence, "
+            "source_diversity and session_url are populated; otherwise poll again shortly."
+        ),
+        params=(path("uid", "DeepSearch job uid returned by deepsearch."),),
+        requires_auth=False,
+    ),
+    Action(
+        name="deepsearch_session",
+        method="GET",
+        path="/tools/deepsearch/{uid}/session",
+        summary="Read a finished DeepSearch report",
+        description=(
+            "Returns the full cited report for a finished DeepSearch: summary, findings, "
+            "gaps, sources and metrics."
+        ),
+        params=(path("uid", "DeepSearch job uid returned by deepsearch."),),
+        requires_auth=False,
+    ),
     Action(
         name="search_users",
         method="GET",

devplacepy/services/jobs/deepsearch/__init__.py

@@ -0,0 +1 @@
+# retoor <retoor@molodetz.nl>

devplacepy/services/jobs/deepsearch/chunking.py

@@ -0,0 +1,49 @@
+# retoor <retoor@molodetz.nl>
+
+from __future__ import annotations
+
+import re
+
+CHUNK_TARGET_CHARS = 1200
+CHUNK_OVERLAP_CHARS = 150
+MIN_CHUNK_CHARS = 80
+PARAGRAPH_SPLIT = re.compile(r"\n\s*\n")
+WHITESPACE = re.compile(r"[ \t]+")
+
+
+def normalize_text(text: str) -> str:
+    cleaned = (text or "").replace("\r", "\n")
+    cleaned = WHITESPACE.sub(" ", cleaned)
+    lines = [line.strip() for line in cleaned.split("\n")]
+    return "\n".join(line for line in lines if line)
+
+
+def chunk_text(text: str) -> list[str]:
+    normalized = normalize_text(text)
+    if not normalized:
+        return []
+    paragraphs = [p.strip() for p in PARAGRAPH_SPLIT.split(normalized) if p.strip()]
+    if not paragraphs:
+        paragraphs = [normalized]
+    chunks: list[str] = []
+    buffer = ""
+    for paragraph in paragraphs:
+        if len(paragraph) > CHUNK_TARGET_CHARS:
+            if buffer:
+                chunks.append(buffer)
+                buffer = ""
+            for start in range(0, len(paragraph), CHUNK_TARGET_CHARS - CHUNK_OVERLAP_CHARS):
+                piece = paragraph[start : start + CHUNK_TARGET_CHARS]
+                if len(piece) >= MIN_CHUNK_CHARS:
+                    chunks.append(piece)
+            continue
+        if len(buffer) + len(paragraph) + 1 > CHUNK_TARGET_CHARS and buffer:
+            chunks.append(buffer)
+            buffer = paragraph
+        else:
+            buffer = f"{buffer}\n{paragraph}" if buffer else paragraph
+    if buffer and len(buffer) >= MIN_CHUNK_CHARS:
+        chunks.append(buffer)
+    elif buffer and not chunks:
+        chunks.append(buffer)
+    return chunks

devplacepy/services/jobs/deepsearch/crawl.py

@@ -0,0 +1,214 @@
+# retoor <retoor@molodetz.nl>
+
+from __future__ import annotations
+
+import hashlib
+import logging
+import re
+from dataclasses import dataclass, field
+from typing import Awaitable, Callable
+
+import httpx
+
+from devplacepy.net_guard import BlockedAddressError, guard_public_url, guarded_async_client
+
+logger = logging.getLogger(__name__)
+
+RSEARCH_URL = "https://rsearch.app.molodetz.nl"
+RSEARCH_TIMEOUT_SECONDS = 45.0
+FETCH_TIMEOUT_SECONDS = 20.0
+MAX_FETCH_BYTES = 2_500_000
+RESULTS_PER_QUERY = 8
+USER_AGENT = (
+    "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) "
+    "Chrome/131.0.0.0 Safari/537.36 DevPlaceDeepSearchBot/1.0"
+)
+SCRIPT_STYLE = re.compile(r"<(script|style)[^>]*>.*?</\1>", re.DOTALL | re.IGNORECASE)
+TAG = re.compile(r"<[^>]+>")
+TITLE = re.compile(r"<title[^>]*>(.*?)</title>", re.DOTALL | re.IGNORECASE)
+SPACE = re.compile(r"\s+")
+MIN_PAGE_CHARS = 200
+
+
+@dataclass
+class CrawledPage:
+    url: str
+    title: str
+    text: str
+    source: str
+    status: int
+    depth: int = 0
+    from_cache: bool = False
+
+
+@dataclass
+class CrawlOutcome:
+    pages: list[CrawledPage] = field(default_factory=list)
+    seen_hashes: set[str] = field(default_factory=set)
+
+
+def url_hash(url: str) -> str:
+    return hashlib.sha256(url.strip().lower().encode("utf-8")).hexdigest()
+
+
+def content_hash(text: str) -> str:
+    return hashlib.sha256(text.strip().encode("utf-8")).hexdigest()
+
+
+def _strip_html(raw: str) -> tuple[str, str]:
+    title_match = TITLE.search(raw)
+    title = SPACE.sub(" ", TAG.sub("", title_match.group(1))).strip() if title_match else ""
+    body = SCRIPT_STYLE.sub(" ", raw)
+    body = TAG.sub(" ", body)
+    body = SPACE.sub(" ", body).strip()
+    return title, body
+
+
+async def search_queries(queries: list[str]) -> list[dict]:
+    results: list[dict] = []
+    seen: set[str] = set()
+    headers = {"User-Agent": USER_AGENT, "Accept": "application/json"}
+    timeout = httpx.Timeout(RSEARCH_TIMEOUT_SECONDS, connect=30.0)
+    async with httpx.AsyncClient(
+        base_url=RSEARCH_URL, headers=headers, follow_redirects=True, timeout=timeout
+    ) as client:
+        for query in queries:
+            try:
+                response = await client.get(
+                    "/search",
+                    params={"query": query, "count": RESULTS_PER_QUERY, "content": "false"},
+                )
+                if response.status_code >= 400:
+                    continue
+                data = response.json()
+            except (httpx.HTTPError, ValueError) as exc:
+                logger.warning("deepsearch rsearch failed for %r: %s", query, exc)
+                continue
+            for item in data.get("results") or []:
+                url = (item.get("url") or "").strip()
+                if not url or url in seen:
+                    continue
+                seen.add(url)
+                results.append(
+                    {
+                        "url": url,
+                        "title": item.get("title") or "",
+                        "description": item.get("description") or "",
+                    }
+                )
+    return results
+
+
+async def _render_with_playwright(url: str) -> tuple[str, str, int]:
+    from playwright.async_api import async_playwright
+
+    async with async_playwright() as pw:
+        browser = await pw.chromium.launch(
+            headless=True, args=["--no-sandbox", "--disable-dev-shm-usage"]
+        )
+        try:
+            context = await browser.new_context(user_agent=USER_AGENT)
+            page = await context.new_page()
+            response = await page.goto(url, wait_until="load", timeout=30000)
+            status = response.status if response else 0
+            for hop in [response.url] if response else []:
+                await guard_public_url(hop)
+            content = (await page.content())[:MAX_FETCH_BYTES]
+            await context.close()
+            title, text = _strip_html(content)
+            return title, text, status
+        finally:
+            await browser.close()
+
+
+async def fetch_page(url: str, depth: int) -> CrawledPage | None:
+    try:
+        await guard_public_url(url)
+    except BlockedAddressError:
+        return None
+    title = ""
+    text = ""
+    status = 0
+    source = "httpx"
+    try:
+        async with guarded_async_client(
+            follow_redirects=True,
+            timeout=FETCH_TIMEOUT_SECONDS,
+            headers={"User-Agent": USER_AGENT},
+        ) as client:
+            response = await client.get(url)
+            await guard_public_url(str(response.url))
+            status = response.status_code
+            raw = response.text[:MAX_FETCH_BYTES]
+            title, text = _strip_html(raw)
+    except (httpx.HTTPError, BlockedAddressError) as exc:
+        logger.info("deepsearch httpx fetch failed for %s: %s", url, exc)
+    if len(text) < MIN_PAGE_CHARS:
+        try:
+            r_title, r_text, r_status = await _render_with_playwright(url)
+            if len(r_text) > len(text):
+                title, text, status, source = (
+                    r_title or title,
+                    r_text,
+                    r_status or status,
+                    "playwright",
+                )
+        except Exception as exc:
+            logger.info("deepsearch render failed for %s: %s", url, exc)
+    if len(text) < MIN_PAGE_CHARS:
+        return None
+    return CrawledPage(
+        url=url, title=title or url, text=text, source=source, status=status, depth=depth
+    )
+
+
+async def crawl(
+    candidates: list[dict],
+    max_pages: int,
+    emit: Callable[[dict], None],
+    is_cached: Callable[[str], bool],
+    should_stop: Callable[[], Awaitable[bool]],
+) -> CrawlOutcome:
+    outcome = CrawlOutcome()
+    fetched = 0
+    total = min(len(candidates), max_pages)
+    for index, candidate in enumerate(candidates):
+        if fetched >= max_pages:
+            break
+        if await should_stop():
+            emit({"type": "stage", "stage": "cancelled", "message": "Crawl cancelled"})
+            break
+        url = candidate["url"]
+        emit(
+            {
+                "type": "progress",
+                "done": fetched,
+                "total": total,
+                "url": url,
+                "message": f"Fetching {url}",
+            }
+        )
+        if is_cached(url):
+            emit({"type": "page_cached", "url": url})
+        page = await fetch_page(url, depth=0)
+        if page is None:
+            emit({"type": "page_skipped", "url": url})
+            continue
+        digest = content_hash(page.text)
+        if digest in outcome.seen_hashes:
+            emit({"type": "page_duplicate", "url": url})
+            continue
+        outcome.seen_hashes.add(digest)
+        outcome.pages.append(page)
+        fetched += 1
+        emit(
+            {
+                "type": "page_loaded",
+                "url": page.url,
+                "title": page.title,
+                "source": page.source,
+                "done": fetched,
+                "total": total,
+            }
+        )
+    return outcome

devplacepy/services/jobs/deepsearch/enhance.py

@@ -0,0 +1,86 @@
+# retoor <retoor@molodetz.nl>
+
+from __future__ import annotations
+
+import json
+import logging
+import re
+
+import httpx
+
+from devplacepy.config import INTERNAL_GATEWAY_URL, INTERNAL_MODEL
+
+logger = logging.getLogger(__name__)
+
+ENHANCE_TIMEOUT_SECONDS = 90.0
+ENHANCE_MAX_TOKENS = 400
+MAX_SUBQUERIES = 6
+
+PLANNER_PROMPT = (
+    "You plan web research. Given a research question, produce a JSON object with one "
+    "key 'queries': an array of 3 to 6 concise, diverse web search queries that together "
+    "cover the question from multiple angles. Return ONLY the JSON object, no prose."
+)
+
+
+def _fallback(query: str) -> list[str]:
+    base = query.strip()
+    variants = [
+        base,
+        f"{base} overview",
+        f"{base} latest",
+        f"{base} explained",
+    ]
+    seen: list[str] = []
+    for variant in variants:
+        if variant and variant not in seen:
+            seen.append(variant)
+    return seen[:MAX_SUBQUERIES]
+
+
+def _parse(text: str) -> list[str]:
+    match = re.search(r"\{.*\}", text, re.DOTALL)
+    if not match:
+        return []
+    try:
+        payload = json.loads(match.group())
+    except (ValueError, TypeError):
+        return []
+    queries = payload.get("queries")
+    if not isinstance(queries, list):
+        return []
+    cleaned = [str(item).strip() for item in queries if str(item).strip()]
+    return cleaned[:MAX_SUBQUERIES]
+
+
+async def plan_queries(query: str, api_key: str) -> list[str]:
+    payload = {
+        "model": INTERNAL_MODEL,
+        "messages": [
+            {"role": "system", "content": PLANNER_PROMPT},
+            {"role": "user", "content": query},
+        ],
+        "max_tokens": ENHANCE_MAX_TOKENS,
+        "temperature": 0.3,
+    }
+    headers = {
+        "Authorization": f"Bearer {api_key}",
+        "Content-Type": "application/json",
+    }
+    try:
+        async with httpx.AsyncClient(timeout=ENHANCE_TIMEOUT_SECONDS) as client:
+            response = await client.post(
+                INTERNAL_GATEWAY_URL, json=payload, headers=headers
+            )
+        if response.status_code >= 400:
+            raise RuntimeError(f"planner gateway returned {response.status_code}")
+        data = response.json()
+        content = (
+            (data.get("choices") or [{}])[0].get("message", {}).get("content") or ""
+        )
+        parsed = _parse(content)
+        if parsed:
+            return parsed
+    except Exception as exc:
+        logger.warning("deepsearch query planner failed, using fallback: %s", exc)
+    return _fallback(query)

devplacepy/services/jobs/deepsearch/orchestrate.py

@@ -0,0 +1,216 @@
+# retoor <retoor@molodetz.nl>
+
+from __future__ import annotations
+
+import json
+import logging
+import re
+from dataclasses import dataclass, field
+from urllib.parse import urlparse
+
+import httpx
+
+from devplacepy.config import INTERNAL_GATEWAY_URL, INTERNAL_MODEL
+
+logger = logging.getLogger(__name__)
+
+AGENT_TIMEOUT_SECONDS = 120.0
+SUMMARY_MAX_TOKENS = 900
+CRITIC_MAX_TOKENS = 600
+LINKER_MAX_TOKENS = 600
+MAX_CONTEXT_CHARS = 11000
+SCORE_MAX = 100
+CONFIDENCE_BASELINE = 0.35
+DIVERSITY_PAGES_PER_DOMAIN = 2.0
+
+
+@dataclass
+class Orchestration:
+    summary: str = ""
+    findings: list[dict] = field(default_factory=list)
+    gaps: list[str] = field(default_factory=list)
+    confidence: float = 0.0
+    source_diversity: float = 0.0
+    score: int = 0
+
+
+def _domain(url: str) -> str:
+    try:
+        return urlparse(url).netloc.lower()
+    except ValueError:
+        return ""
+
+
+def source_diversity(pages: list) -> float:
+    if not pages:
+        return 0.0
+    domains = {_domain(page.url) for page in pages if getattr(page, "url", "")}
+    domains.discard("")
+    if not domains:
+        return 0.0
+    ratio = len(domains) / max(1.0, len(pages) / DIVERSITY_PAGES_PER_DOMAIN)
+    return round(min(1.0, ratio), 3)
+
+
+def _build_context(pages: list) -> str:
+    blocks: list[str] = []
+    used = 0
+    for index, page in enumerate(pages, start=1):
+        snippet = (page.text or "")[:1600]
+        block = f"[{index}] {page.title} ({page.url})\n{snippet}"
+        if used + len(block) > MAX_CONTEXT_CHARS and blocks:
+            break
+        used += len(block)
+        blocks.append(block)
+    return "\n\n".join(blocks)
+
+
+async def _complete(messages: list[dict], api_key: str, max_tokens: int) -> str:
+    payload = {
+        "model": INTERNAL_MODEL,
+        "messages": messages,
+        "max_tokens": max_tokens,
+        "temperature": 0.2,
+    }
+    headers = {"Authorization": f"Bearer {api_key}", "Content-Type": "application/json"}
+    async with httpx.AsyncClient(timeout=AGENT_TIMEOUT_SECONDS) as client:
+        response = await client.post(INTERNAL_GATEWAY_URL, json=payload, headers=headers)
+    if response.status_code >= 400:
+        raise RuntimeError(f"agent gateway returned {response.status_code}")
+    data = response.json()
+    return (data.get("choices") or [{}])[0].get("message", {}).get("content") or ""
+
+
+def _parse_json(text: str) -> dict:
+    match = re.search(r"\{.*\}", text, re.DOTALL)
+    if not match:
+        return {}
+    try:
+        return json.loads(match.group())
+    except (ValueError, TypeError):
+        return {}
+
+
+SUMMARIZER_PROMPT = (
+    "You are a research summarizer. Using ONLY the numbered SOURCES, write a JSON object "
+    "with keys: 'summary' (a grounded markdown summary answering the question) and "
+    "'findings' (an array of objects, each with 'title', 'detail', 'confidence' between 0 "
+    "and 1, and 'citations' an array of source numbers). Cite only the provided sources. "
+    "Return ONLY the JSON object."
+)
+CRITIC_PROMPT = (
+    "You are a research critic. Given a QUESTION, a draft SUMMARY and FINDINGS, identify "
+    "what is missing, contradictory, or weakly supported. Return ONLY a JSON object with "
+    "key 'gaps': an array of short strings describing open questions or weak spots."
+)
+LINKER_PROMPT = (
+    "You are a research linker. Given FINDINGS and the SOURCES, refine the confidence of "
+    "each finding based on how many independent sources support it. Return ONLY a JSON "
+    "object with key 'confidence': a number between 0 and 1 estimating overall answer "
+    "confidence given source agreement and coverage."
+)
+
+
+def _heuristic(question: str, pages: list) -> Orchestration:
+    diversity = source_diversity(pages)
+    findings = []
+    for page in pages[:5]:
+        findings.append(
+            {
+                "title": page.title[:120] or page.url,
+                "detail": (page.text or "")[:400],
+                "confidence": round(min(0.6, CONFIDENCE_BASELINE + diversity / 4), 3),
+                "citations": [page.url],
+            }
+        )
+    summary = (
+        f"Collected {len(pages)} sources for '{question}'. Automatic synthesis was "
+        "unavailable, so the top findings are listed verbatim from the gathered sources."
+    )
+    confidence = round(min(0.6, CONFIDENCE_BASELINE + diversity / 3), 3)
+    score = int(min(SCORE_MAX, len(pages) * 6 + diversity * 30))
+    return Orchestration(
+        summary=summary,
+        findings=findings,
+        gaps=["Automatic critique was unavailable for this run."],
+        confidence=confidence,
+        source_diversity=diversity,
+        score=score,
+    )
+
+
+async def orchestrate(question: str, pages: list, api_key: str, emit) -> Orchestration:
+    diversity = source_diversity(pages)
+    if not pages:
+        return Orchestration(gaps=["No sources were gathered."], source_diversity=0.0)
+    context = _build_context(pages)
+    try:
+        emit({"type": "agent", "agent": "summarizer", "message": "Synthesising findings"})
+        summary_raw = await _complete(
+            [
+                {"role": "system", "content": SUMMARIZER_PROMPT},
+                {
+                    "role": "user",
+                    "content": f"QUESTION: {question}\n\nSOURCES:\n{context}",
+                },
+            ],
+            api_key,
+            SUMMARY_MAX_TOKENS,
+        )
+        parsed = _parse_json(summary_raw)
+        summary = str(parsed.get("summary", "")).strip()
+        findings = [f for f in (parsed.get("findings") or []) if isinstance(f, dict)]
+        if not summary and not findings:
+            return _heuristic(question, pages)
+
+        emit({"type": "agent", "agent": "critic", "message": "Reviewing for gaps"})
+        gaps_raw = await _complete(
+            [
+                {"role": "system", "content": CRITIC_PROMPT},
+                {
+                    "role": "user",
+                    "content": (
+                        f"QUESTION: {question}\n\nSUMMARY: {summary}\n\n"
+                        f"FINDINGS: {json.dumps(findings)[:4000]}"
+                    ),
+                },
+            ],
+            api_key,
+            CRITIC_MAX_TOKENS,
+        )
+        gaps = [str(g).strip() for g in (_parse_json(gaps_raw).get("gaps") or []) if str(g).strip()]
+
+        emit({"type": "agent", "agent": "linker", "message": "Scoring confidence"})
+        link_raw = await _complete(
+            [
+                {"role": "system", "content": LINKER_PROMPT},
+                {
+                    "role": "user",
+                    "content": (
+                        f"FINDINGS: {json.dumps(findings)[:4000]}\n\nSOURCES:\n{context[:4000]}"
+                    ),
+                },
+            ],
+            api_key,
+            LINKER_MAX_TOKENS,
+        )
+        try:
+            confidence = float(_parse_json(link_raw).get("confidence", 0.0))
+        except (TypeError, ValueError):
+            confidence = 0.0
+        confidence = round(max(CONFIDENCE_BASELINE, min(1.0, confidence)), 3)
+        coverage = min(1.0, len(pages) / 10.0)
+        score = int(
+            min(SCORE_MAX, (confidence * 0.5 + diversity * 0.3 + coverage * 0.2) * SCORE_MAX)
+        )
+        return Orchestration(
+            summary=summary,
+            findings=findings,
+            gaps=gaps,
+            confidence=confidence,
+            source_diversity=diversity,
+            score=score,
+        )
+    except Exception as exc:
+        logger.warning("deepsearch orchestration failed, using heuristic: %s", exc)
+        return _heuristic(question, pages)

devplacepy/services/jobs/deepsearch/progress.py

@@ -0,0 +1,43 @@
+# retoor <retoor@molodetz.nl>
+
+from __future__ import annotations
+
+import asyncio
+
+MAX_BUFFER = 2000
+
+
+class ProgressHub:
+    def __init__(self) -> None:
+        self._subscribers: dict[str, set[asyncio.Queue]] = {}
+        self._buffers: dict[str, list[dict]] = {}
+
+    def subscribe(self, uid: str) -> asyncio.Queue:
+        queue: asyncio.Queue = asyncio.Queue()
+        self._subscribers.setdefault(uid, set()).add(queue)
+        return queue
+
+    def unsubscribe(self, uid: str, queue: asyncio.Queue) -> None:
+        listeners = self._subscribers.get(uid)
+        if not listeners:
+            return
+        listeners.discard(queue)
+        if not listeners:
+            self._subscribers.pop(uid, None)
+
+    def publish(self, uid: str, frame: dict) -> None:
+        buffer = self._buffers.setdefault(uid, [])
+        buffer.append(frame)
+        if len(buffer) > MAX_BUFFER:
+            del buffer[: len(buffer) - MAX_BUFFER]
+        for queue in self._subscribers.get(uid, set()):
+            queue.put_nowait(frame)
+
+    def snapshot(self, uid: str) -> list[dict]:
+        return list(self._buffers.get(uid, []))
+
+    def clear(self, uid: str) -> None:
+        self._buffers.pop(uid, None)
+
+
+hub = ProgressHub()

devplacepy/services/jobs/deepsearch/service.py

@@ -0,0 +1,212 @@
+# retoor <retoor@molodetz.nl>
+
+from __future__ import annotations
+
+import asyncio
+import json
+import logging
+import shutil
+import sys
+from pathlib import Path
+
+from devplacepy.config import BASE_DIR, DEEPSEARCH_DIR
+from devplacepy.services.deepsearch.store import VectorStore
+from devplacepy.services.jobs.base import JobService
+
+from .progress import hub
+
+logger = logging.getLogger(__name__)
+
+WORKER_MODULE = "devplacepy.services.jobs.deepsearch.worker"
+STREAM_LIMIT = 16 * 1024 * 1024
+
+
+class DeepsearchService(JobService):
+    kind = "deepsearch"
+    title = "DeepSearch"
+    description = (
+        "Runs a multi-agent web research job: it plans queries, crawls and indexes "
+        "sources into a per-session vector collection, then synthesises a cited report "
+        "with confidence scoring, source diversity and gap analysis, streaming live "
+        "progress over a websocket."
+    )
+
+    def __init__(self):
+        super().__init__(name="deepsearch", interval_seconds=2)
+
+    def session_dir(self, uid: str) -> Path:
+        return DEEPSEARCH_DIR / uid
+
+    def collection_name(self, uid: str) -> str:
+        return f"ds_{uid.replace('-', '')}"
+
+    async def process(self, job: dict) -> dict:
+        from devplacepy import database
+        from devplacepy.services.audit import record as audit
+
+        uid = job["uid"]
+        payload = dict(job.get("payload", {}))
+        query = payload.get("query", "")
+        output_dir = self.session_dir(uid)
+        output_dir.mkdir(parents=True, exist_ok=True)
+        (output_dir / "control.json").write_text(
+            json.dumps({"state": "running"}), encoding="utf-8"
+        )
+        payload["collection"] = self.collection_name(uid)
+        payload["cached_hashes"] = self._cached_hashes(database)
+        payload_path = output_dir / "payload.json"
+        payload_path.write_text(json.dumps(payload), encoding="utf-8")
+
+        actor_kind = job.get("owner_kind") or "system"
+        actor_uid = job.get("owner_id") if job.get("owner_kind") == "user" else None
+        database.update_deepsearch_session(uid, {"status": "running"})
+
+        try:
+            summary = await self._run_worker(uid, payload_path, output_dir)
+        except Exception as exc:
+            hub.publish(uid, {"type": "failed", "message": str(exc)[:300]})
+            database.update_deepsearch_session(uid, {"status": "failed"})
+            audit.record_system(
+                "deepsearch.run.failed",
+                actor_kind=actor_kind,
+                actor_uid=actor_uid,
+                result="failure",
+                summary=f"DeepSearch for {query} failed",
+                metadata={"query": query, "error": str(exc)[:200]},
+                links=[audit.job(uid)],
+            )
+            raise
+
+        report = self._load_report(output_dir)
+        self._persist_cache(database, output_dir)
+        from datetime import datetime, timezone
+
+        database.update_deepsearch_session(
+            uid,
+            {
+                "status": "done",
+                "score": int(summary.get("score") or 0),
+                "confidence": float(summary.get("confidence") or 0.0),
+                "source_diversity": float(summary.get("source_diversity") or 0.0),
+                "page_count": int(summary.get("page_count") or 0),
+                "chunk_count": int(summary.get("chunk_count") or 0),
+                "summary": (report.get("summary") or "")[:4000],
+                "completed_at": datetime.now(timezone.utc).isoformat(),
+            },
+        )
+        hub.publish(
+            uid,
+            {
+                "type": "done",
+                "score": summary.get("score"),
+                "confidence": summary.get("confidence"),
+                "source_diversity": summary.get("source_diversity"),
+                "page_count": summary.get("page_count"),
+                "chunk_count": summary.get("chunk_count"),
+                "session_url": f"/tools/deepsearch/{uid}/session",
+            },
+        )
+        hub.clear(uid)
+        audit.record_system(
+            "deepsearch.run.complete",
+            actor_kind=actor_kind,
+            actor_uid=actor_uid,
+            summary=f"DeepSearch for {query} scored {summary.get('score')}",
+            metadata={
+                "query": query,
+                "score": summary.get("score"),
+                "page_count": summary.get("page_count"),
+            },
+            links=[audit.job(uid)],
+        )
+        return {
+            "query": query,
+            "score": summary.get("score", 0),
+            "confidence": summary.get("confidence", 0.0),
+            "source_diversity": summary.get("source_diversity", 0.0),
+            "page_count": summary.get("page_count", 0),
+            "chunk_count": summary.get("chunk_count", 0),
+            "report": report,
+            "collection": self.collection_name(uid),
+            "session_url": f"/tools/deepsearch/{uid}/session",
+            "bytes_in": 0,
+            "bytes_out": len(json.dumps(report)) if report else 0,
+            "item_count": summary.get("page_count", 0),
+        }
+
+    def _cached_hashes(self, database) -> list[str]:
+        if "deepsearch_url_cache" not in database.db.tables:
+            return []
+        return [
+            row.get("url_hash", "")
+            for row in database.get_table("deepsearch_url_cache").find()
+            if row.get("url_hash")
+        ]
+
+    def _persist_cache(self, database, output_dir: Path) -> None:
+        path = output_dir / "url_cache.json"
+        if not path.is_file():
+            return
+        try:
+            entries = json.loads(path.read_text(encoding="utf-8"))
+        except (ValueError, OSError):
+            return
+        for entry in entries:
+            database.upsert_deepsearch_url_cache(
+                entry.get("url_hash", ""),
+                entry.get("url", ""),
+                entry.get("title", ""),
+                entry.get("content_hash", ""),
+                int(entry.get("status") or 0),
+                int(entry.get("byte_size") or 0),
+            )
+
+    async def _run_worker(self, uid: str, payload_path: Path, output_dir: Path) -> dict:
+        proc = await asyncio.create_subprocess_exec(
+            sys.executable,
+            "-m",
+            WORKER_MODULE,
+            str(payload_path),
+            str(output_dir),
+            cwd=str(BASE_DIR),
+            stdout=asyncio.subprocess.PIPE,
+            stderr=asyncio.subprocess.PIPE,
+            limit=STREAM_LIMIT,
+        )
+        summary: dict = {}
+        worker_error = ""
+        while True:
+            line = await proc.stdout.readline()
+            if not line:
+                break
+            try:
+                frame = json.loads(line.decode("utf-8", "replace"))
+            except (ValueError, TypeError):
+                continue
+            hub.publish(uid, frame)
+            if frame.get("type") == "report_ready":
+                summary = frame
+            elif frame.get("type") == "error":
+                worker_error = frame.get("message", "worker error")
+        err = (await proc.stderr.read()).decode("utf-8", "replace")
+        await proc.wait()
+        if proc.returncode != 0 or not summary:
+            raise RuntimeError(
+                worker_error or err[:500] or f"deepsearch worker exited {proc.returncode}"
+            )
+        return summary
+
+    def _load_report(self, output_dir: Path) -> dict:
+        report_path = output_dir / "report.json"
+        if not report_path.is_file():
+            return {}
+        try:
+            return json.loads(report_path.read_text(encoding="utf-8"))
+        except (ValueError, OSError):
+            return {}
+
+    def cleanup(self, job: dict) -> None:
+        uid = job["uid"]
+        hub.clear(uid)
+        VectorStore(self.collection_name(uid)).drop()
+        shutil.rmtree(self.session_dir(uid), ignore_errors=True)

devplacepy/services/jobs/deepsearch/worker.py

@@ -0,0 +1,186 @@
+# retoor <retoor@molodetz.nl>
+
+from __future__ import annotations
+
+import asyncio
+import json
+import sys
+from datetime import datetime, timezone
+from pathlib import Path
+
+from devplacepy.services.deepsearch.embeddings import embed_texts, local_embed
+from devplacepy.services.deepsearch.store import Chunk, VectorStore
+from devplacepy.utils import generate_uid
+
+from .chunking import chunk_text
+from .crawl import content_hash, crawl, search_queries, url_hash
+from .enhance import plan_queries
+from .orchestrate import orchestrate
+
+CONTROL_FILE = "control.json"
+PAUSE_POLL_SECONDS = 1.0
+EMBED_BATCH = 64
+
+
+def _emit(frame: dict) -> None:
+    sys.stdout.write(json.dumps(frame, ensure_ascii=False) + "\n")
+    sys.stdout.flush()
+
+
+def _read_control(output_dir: Path) -> str:
+    path = output_dir / CONTROL_FILE
+    if not path.is_file():
+        return "running"
+    try:
+        return str(json.loads(path.read_text(encoding="utf-8")).get("state", "running"))
+    except (ValueError, OSError):
+        return "running"
+
+
+def _make_stop(output_dir: Path):
+    async def should_stop() -> bool:
+        while True:
+            state = _read_control(output_dir)
+            if state == "cancelled":
+                return True
+            if state == "paused":
+                _emit({"type": "stage", "stage": "paused", "message": "Paused"})
+                await asyncio.sleep(PAUSE_POLL_SECONDS)
+                continue
+            return False
+
+    return should_stop
+
+
+async def _index_chunks(
+    store: VectorStore, pages: list, api_key: str
+) -> tuple[int, str]:
+    chunks: list[Chunk] = []
+    for page in pages:
+        for position, text in enumerate(chunk_text(page.text)):
+            chunks.append(
+                Chunk(
+                    uid=generate_uid(),
+                    text=text,
+                    url=page.url,
+                    title=page.title,
+                    depth=page.depth,
+                    source=page.source,
+                    position=position,
+                )
+            )
+    if not chunks:
+        return 0, "empty"
+    backend = "gateway"
+    for start in range(0, len(chunks), EMBED_BATCH):
+        batch = chunks[start : start + EMBED_BATCH]
+        result = await embed_texts([chunk.text for chunk in batch], api_key)
+        if not result.vectors:
+            result = local_embed([chunk.text for chunk in batch])
+        backend = result.backend
+        store.add(batch, result.vectors)
+    return len(chunks), backend
+
+
+async def _run(payload: dict, output_dir: Path) -> dict:
+    query = payload.get("query", "")
+    max_pages = int(payload.get("max_pages", 12))
+    depth = int(payload.get("depth", 2))
+    api_key = payload.get("api_key", "")
+    collection = payload.get("collection", "")
+    cached_hashes = set(payload.get("cached_hashes", []))
+    should_stop = _make_stop(output_dir)
+
+    _emit({"type": "stage", "stage": "planning", "message": "Planning research queries"})
+    queries = await plan_queries(query, api_key)
+    _emit({"type": "queries", "queries": queries})
+
+    _emit({"type": "stage", "stage": "searching", "message": "Searching the web"})
+    candidates = await search_queries(queries)
+    _emit({"type": "candidates", "count": len(candidates)})
+
+    _emit({"type": "stage", "stage": "crawling", "message": "Crawling sources"})
+    outcome = await crawl(
+        candidates,
+        max_pages,
+        _emit,
+        lambda url: url_hash(url) in cached_hashes,
+        should_stop,
+    )
+
+    new_cache = [
+        {
+            "url_hash": url_hash(page.url),
+            "url": page.url,
+            "title": page.title,
+            "content_hash": content_hash(page.text),
+            "status": page.status,
+            "byte_size": len(page.text),
+        }
+        for page in outcome.pages
+    ]
+
+    store = VectorStore(collection)
+    _emit({"type": "stage", "stage": "indexing", "message": "Indexing content"})
+    chunk_count, embed_backend = await _index_chunks(store, outcome.pages, api_key)
+
+    _emit({"type": "stage", "stage": "analysis", "message": "Running research agents"})
+    result = await orchestrate(query, outcome.pages, api_key, _emit)
+
+    sources = [
+        {"url": page.url, "title": page.title, "source": page.source}
+        for page in outcome.pages
+    ]
+    report = {
+        "query": query,
+        "depth": depth,
+        "max_pages": max_pages,
+        "generated_at": datetime.now(timezone.utc).isoformat(),
+        "summary": result.summary,
+        "findings": result.findings,
+        "gaps": result.gaps,
+        "sources": sources,
+        "score": result.score,
+        "confidence": result.confidence,
+        "source_diversity": result.source_diversity,
+        "page_count": len(outcome.pages),
+        "chunk_count": chunk_count,
+        "embed_backend": embed_backend,
+        "collection": collection,
+    }
+    (output_dir / "report.json").write_text(
+        json.dumps(report, ensure_ascii=False), encoding="utf-8"
+    )
+    (output_dir / "url_cache.json").write_text(
+        json.dumps(new_cache, ensure_ascii=False), encoding="utf-8"
+    )
+    _emit(
+        {
+            "type": "report_ready",
+            "score": report["score"],
+            "confidence": report["confidence"],
+            "source_diversity": report["source_diversity"],
+            "page_count": report["page_count"],
+            "chunk_count": report["chunk_count"],
+        }
+    )
+    return report
+
+
+def main(argv: list) -> int:
+    if len(argv) != 3:
+        sys.stderr.write("usage: deepsearch.worker <payload_json> <output_dir>\n")
+        return 2
+    payload = json.loads(Path(argv[1]).read_text(encoding="utf-8"))
+    output_dir = Path(argv[2])
+    output_dir.mkdir(parents=True, exist_ok=True)
+    try:
+        asyncio.run(_run(payload, output_dir))
+    except Exception as exc:
+        _emit({"type": "error", "message": str(exc)[:500]})
+        return 1
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main(sys.argv))

devplacepy/services/openai_gateway/config.py

@@ -6,6 +6,8 @@ TIMEOUT_DEFAULT = 300
 TIMEOUT_MIN = 300
 INSTANCES_DEFAULT = 4
 
+SYSTEM_PREAMBLE_DEFAULT = ""
+
 VISION_URL_DEFAULT = "https://openrouter.ai/api/v1/chat/completions"
 VISION_MODEL_DEFAULT = "google/gemma-3-12b-it"
 VISION_CACHE_SIZE_DEFAULT = 256

devplacepy/services/openai_gateway/gateway.py

@@ -12,6 +12,7 @@ from fastapi.responses import JSONResponse, Response, StreamingResponse
 
 from devplacepy.services.openai_gateway import config
 from devplacepy.services.openai_gateway.reliability import CircuitBreaker, retry_send
+from devplacepy.services.openai_gateway.system_message import apply_system_directives
 from devplacepy.services.openai_gateway.usage import (
     GatewayUsageLedger,
     classify_error,
@@ -229,6 +230,8 @@ class GatewayRuntime:
             messages = await augmenter.augment_messages(client, messages)
             self.vision_calls += augmenter.calls
 
+        messages = apply_system_directives(messages, cfg.get("gateway_system_preamble", ""))
+
         requested = body.get("model")
         if cfg["gateway_force_model"] or not requested or requested == "molodetz":
             model = cfg["gateway_model"]
@@ -351,6 +354,25 @@ class GatewayRuntime:
     ):
         log = log or (lambda message: None)
         if not cfg["gateway_embed_enabled"]:
+            from devplacepy.services.audit import record as audit
+            from devplacepy.services.openai_gateway.usage import audit_actor_for
+
+            actor_kind, actor_uid, actor_role = audit_actor_for(owner[0], owner[1])
+            audit.record_system(
+                "ai.gateway.call",
+                actor_kind=actor_kind,
+                actor_uid=actor_uid,
+                actor_role=actor_role,
+                origin="api",
+                result="denied",
+                summary="embeddings disabled",
+                metadata={
+                    "backend": "embed",
+                    "endpoint": "embeddings",
+                    "owner_kind": owner[0],
+                    "owner_id": owner[1],
+                },
+            )
             return JSONResponse(
                 status_code=503,
                 content={

devplacepy/services/openai_gateway/service.py

@@ -89,6 +89,16 @@ class GatewayService(BaseService):
             help="Max concurrent upstream forwards per worker (connection pool + semaphore).",
             group="Upstream",
         ),
+        ConfigField(
+            "gateway_system_preamble",
+            "System preamble",
+            type="text",
+            default=config.SYSTEM_PREAMBLE_DEFAULT,
+            help="Operator text prepended ahead of every chat request's system message "
+            "(before an auto-injected EU-format date line and the client's own system "
+            "content, all in one system message). Leave blank to disable.",
+            group="Prompt",
+        ),
         ConfigField(
             "gateway_vision_enabled",
             "Vision augmentation",

devplacepy/services/openai_gateway/system_message.py

@@ -0,0 +1,110 @@
+# retoor <retoor@molodetz.nl>
+
+import logging
+import re
+from datetime import datetime
+from typing import Any, Optional
+
+logger = logging.getLogger(__name__)
+
+DATE_LABEL = "Current date"
+
+_MONTH_NAMES = (
+    "january|february|march|april|may|june|july|august|september|october|"
+    "november|december|jan|feb|mar|apr|jun|jul|aug|sep|sept|oct|nov|dec"
+)
+
+_DATE_PATTERNS: tuple[re.Pattern[str], ...] = (
+    re.compile(r"\b\d{4}-\d{1,2}-\d{1,2}\b"),
+    re.compile(r"\b\d{1,2}/\d{1,2}/\d{4}\b"),
+    re.compile(r"\b\d{1,2}-\d{1,2}-\d{4}\b"),
+    re.compile(r"\b\d{1,2}\.\d{1,2}\.\d{4}\b"),
+    re.compile(
+        rf"\b\d{{1,2}}\s+(?:{_MONTH_NAMES})\.?\s+\d{{4}}\b",
+        re.IGNORECASE,
+    ),
+    re.compile(
+        rf"\b(?:{_MONTH_NAMES})\.?\s+\d{{1,2}}(?:st|nd|rd|th)?,?\s+\d{{4}}\b",
+        re.IGNORECASE,
+    ),
+)
+
+
+def contains_date(text: str) -> bool:
+    return any(pattern.search(text) for pattern in _DATE_PATTERNS)
+
+
+def current_date_eu() -> str:
+    return datetime.now().strftime("%d/%m/%Y")
+
+
+def system_message_text(content: Any) -> str:
+    if isinstance(content, str):
+        return content
+    if isinstance(content, list):
+        parts: list[str] = []
+        for block in content:
+            if isinstance(block, dict) and block.get("type") == "text":
+                value = block.get("text")
+                if isinstance(value, str):
+                    parts.append(value)
+        return "\n".join(parts)
+    return ""
+
+
+def compose_system_content(
+    preamble: str, client_content: Any, date_eu: str
+) -> str:
+    preamble_text = (preamble or "").strip()
+    client_text = system_message_text(client_content)
+    sections: list[str] = []
+    if preamble_text:
+        sections.append(preamble_text)
+    combined_for_date = f"{preamble_text}\n{client_text}"
+    if not contains_date(combined_for_date):
+        sections.append(f"{DATE_LABEL}: {date_eu}")
+    if client_text:
+        sections.append(client_text)
+    return "\n\n".join(sections)
+
+
+def apply_system_directives(
+    messages: list, preamble: str, date_eu: Optional[str] = None
+) -> list:
+    date_value = date_eu or current_date_eu()
+    preamble_text = (preamble or "").strip()
+    result: list = list(messages)
+    system_index = next(
+        (
+            index
+            for index, message in enumerate(result)
+            if isinstance(message, dict) and message.get("role") == "system"
+        ),
+        None,
+    )
+    if system_index is not None:
+        original = result[system_index]
+        composed = compose_system_content(
+            preamble_text, original.get("content"), date_value
+        )
+        updated = dict(original)
+        updated["content"] = composed
+        result[system_index] = updated
+        logger.debug(
+            "Gateway composed existing system message (preamble=%s, date=%s)",
+            bool(preamble_text),
+            date_value,
+        )
+        return result
+    if not preamble_text:
+        return result
+    composed = compose_system_content(preamble_text, "", date_value)
+    if not composed.strip():
+        return result
+    result.insert(0, {"role": "system", "content": composed})
+    logger.info(
+        "Gateway injected a system message (preamble=%s, date=%s)",
+        bool(preamble_text),
+        date_value,
+    )
+    return result

devplacepy/services/openai_gateway/usage.py

@@ -204,6 +204,21 @@ def classify_error(
     return "gateway"
 
 
+def audit_actor_for(owner_kind: str, owner_id: str) -> tuple[str, Optional[str], str]:
+    actor_kind = (
+        "guest"
+        if owner_kind == "guest"
+        else ("user" if owner_kind in ("user", "admin") else "system")
+    )
+    actor_uid = owner_id if actor_kind == "user" else None
+    actor_role = (
+        "admin"
+        if owner_kind == "admin"
+        else (actor_kind if actor_kind != "user" else "member")
+    )
+    return actor_kind, actor_uid, actor_role
+
+
 class GatewayUsageLedger:
     def record(self, raw: dict, pricing: Pricing, context_map: dict) -> None:
         try:
@@ -271,12 +286,12 @@ class GatewayUsageLedger:
 
         owner_kind = raw.get("owner_kind") or "unknown"
         owner_id = raw.get("owner_id") or "unknown"
-        actor_kind = "guest" if owner_kind == "guest" else ("user" if owner_kind in ("user", "admin") else "system")
+        actor_kind, actor_uid, actor_role = audit_actor_for(owner_kind, owner_id)
         audit.record_system(
             "ai.gateway.call",
             actor_kind=actor_kind,
-            actor_uid=owner_id if actor_kind == "user" else None,
-            actor_role="admin" if owner_kind == "admin" else (actor_kind if actor_kind != "user" else "member"),
+            actor_uid=actor_uid,
+            actor_role=actor_role,
             origin="api",
             result="success" if raw.get("success") else "failure",
             summary=f"LLM call by {owner_kind}/{owner_id} (model {raw.get('model') or ''})",