feat: add deepsearch research system with CLI prune/clear and database schema

Implement a multi-agent deep web research subsystem including CLI commands for pruning expired jobs and clearing all artifacts, database tables for sessions/messages/URL cache with indexes, config paths for chroma storage, and internal embed URL for vector operations.

devplacepy/services/openai_gateway/config.py

@@ -6,6 +6,8 @@ TIMEOUT_DEFAULT = 300
 TIMEOUT_MIN = 300
 INSTANCES_DEFAULT = 4
 
+SYSTEM_PREAMBLE_DEFAULT = ""
+
 VISION_URL_DEFAULT = "https://openrouter.ai/api/v1/chat/completions"
 VISION_MODEL_DEFAULT = "google/gemma-3-12b-it"
 VISION_CACHE_SIZE_DEFAULT = 256

devplacepy/services/openai_gateway/gateway.py

@@ -12,6 +12,7 @@ from fastapi.responses import JSONResponse, Response, StreamingResponse
 
 from devplacepy.services.openai_gateway import config
 from devplacepy.services.openai_gateway.reliability import CircuitBreaker, retry_send
+from devplacepy.services.openai_gateway.system_message import apply_system_directives
 from devplacepy.services.openai_gateway.usage import (
     GatewayUsageLedger,
     classify_error,
@@ -229,6 +230,8 @@ class GatewayRuntime:
             messages = await augmenter.augment_messages(client, messages)
             self.vision_calls += augmenter.calls
 
+        messages = apply_system_directives(messages, cfg.get("gateway_system_preamble", ""))
+
         requested = body.get("model")
         if cfg["gateway_force_model"] or not requested or requested == "molodetz":
             model = cfg["gateway_model"]
@@ -351,6 +354,25 @@ class GatewayRuntime:
     ):
         log = log or (lambda message: None)
         if not cfg["gateway_embed_enabled"]:
+            from devplacepy.services.audit import record as audit
+            from devplacepy.services.openai_gateway.usage import audit_actor_for
+
+            actor_kind, actor_uid, actor_role = audit_actor_for(owner[0], owner[1])
+            audit.record_system(
+                "ai.gateway.call",
+                actor_kind=actor_kind,
+                actor_uid=actor_uid,
+                actor_role=actor_role,
+                origin="api",
+                result="denied",
+                summary="embeddings disabled",
+                metadata={
+                    "backend": "embed",
+                    "endpoint": "embeddings",
+                    "owner_kind": owner[0],
+                    "owner_id": owner[1],
+                },
+            )
             return JSONResponse(
                 status_code=503,
                 content={

devplacepy/services/openai_gateway/service.py

@@ -89,6 +89,16 @@ class GatewayService(BaseService):
             help="Max concurrent upstream forwards per worker (connection pool + semaphore).",
             group="Upstream",
         ),
+        ConfigField(
+            "gateway_system_preamble",
+            "System preamble",
+            type="text",
+            default=config.SYSTEM_PREAMBLE_DEFAULT,
+            help="Operator text prepended ahead of every chat request's system message "
+            "(before an auto-injected EU-format date line and the client's own system "
+            "content, all in one system message). Leave blank to disable.",
+            group="Prompt",
+        ),
         ConfigField(
             "gateway_vision_enabled",
             "Vision augmentation",

devplacepy/services/openai_gateway/system_message.py

@@ -0,0 +1,110 @@
+# retoor <retoor@molodetz.nl>
+
+import logging
+import re
+from datetime import datetime
+from typing import Any, Optional
+
+logger = logging.getLogger(__name__)
+
+DATE_LABEL = "Current date"
+
+_MONTH_NAMES = (
+    "january|february|march|april|may|june|july|august|september|october|"
+    "november|december|jan|feb|mar|apr|jun|jul|aug|sep|sept|oct|nov|dec"
+)
+
+_DATE_PATTERNS: tuple[re.Pattern[str], ...] = (
+    re.compile(r"\b\d{4}-\d{1,2}-\d{1,2}\b"),
+    re.compile(r"\b\d{1,2}/\d{1,2}/\d{4}\b"),
+    re.compile(r"\b\d{1,2}-\d{1,2}-\d{4}\b"),
+    re.compile(r"\b\d{1,2}\.\d{1,2}\.\d{4}\b"),
+    re.compile(
+        rf"\b\d{{1,2}}\s+(?:{_MONTH_NAMES})\.?\s+\d{{4}}\b",
+        re.IGNORECASE,
+    ),
+    re.compile(
+        rf"\b(?:{_MONTH_NAMES})\.?\s+\d{{1,2}}(?:st|nd|rd|th)?,?\s+\d{{4}}\b",
+        re.IGNORECASE,
+    ),
+)
+
+
+def contains_date(text: str) -> bool:
+    return any(pattern.search(text) for pattern in _DATE_PATTERNS)
+
+
+def current_date_eu() -> str:
+    return datetime.now().strftime("%d/%m/%Y")
+
+
+def system_message_text(content: Any) -> str:
+    if isinstance(content, str):
+        return content
+    if isinstance(content, list):
+        parts: list[str] = []
+        for block in content:
+            if isinstance(block, dict) and block.get("type") == "text":
+                value = block.get("text")
+                if isinstance(value, str):
+                    parts.append(value)
+        return "\n".join(parts)
+    return ""
+
+
+def compose_system_content(
+    preamble: str, client_content: Any, date_eu: str
+) -> str:
+    preamble_text = (preamble or "").strip()
+    client_text = system_message_text(client_content)
+    sections: list[str] = []
+    if preamble_text:
+        sections.append(preamble_text)
+    combined_for_date = f"{preamble_text}\n{client_text}"
+    if not contains_date(combined_for_date):
+        sections.append(f"{DATE_LABEL}: {date_eu}")
+    if client_text:
+        sections.append(client_text)
+    return "\n\n".join(sections)
+
+
+def apply_system_directives(
+    messages: list, preamble: str, date_eu: Optional[str] = None
+) -> list:
+    date_value = date_eu or current_date_eu()
+    preamble_text = (preamble or "").strip()
+    result: list = list(messages)
+    system_index = next(
+        (
+            index
+            for index, message in enumerate(result)
+            if isinstance(message, dict) and message.get("role") == "system"
+        ),
+        None,
+    )
+    if system_index is not None:
+        original = result[system_index]
+        composed = compose_system_content(
+            preamble_text, original.get("content"), date_value
+        )
+        updated = dict(original)
+        updated["content"] = composed
+        result[system_index] = updated
+        logger.debug(
+            "Gateway composed existing system message (preamble=%s, date=%s)",
+            bool(preamble_text),
+            date_value,
+        )
+        return result
+    if not preamble_text:
+        return result
+    composed = compose_system_content(preamble_text, "", date_value)
+    if not composed.strip():
+        return result
+    result.insert(0, {"role": "system", "content": composed})
+    logger.info(
+        "Gateway injected a system message (preamble=%s, date=%s)",
+        bool(preamble_text),
+        date_value,
+    )
+    return result

devplacepy/services/openai_gateway/usage.py

@@ -204,6 +204,21 @@ def classify_error(
     return "gateway"
 
 
+def audit_actor_for(owner_kind: str, owner_id: str) -> tuple[str, Optional[str], str]:
+    actor_kind = (
+        "guest"
+        if owner_kind == "guest"
+        else ("user" if owner_kind in ("user", "admin") else "system")
+    )
+    actor_uid = owner_id if actor_kind == "user" else None
+    actor_role = (
+        "admin"
+        if owner_kind == "admin"
+        else (actor_kind if actor_kind != "user" else "member")
+    )
+    return actor_kind, actor_uid, actor_role
+
+
 class GatewayUsageLedger:
     def record(self, raw: dict, pricing: Pricing, context_map: dict) -> None:
         try:
@@ -271,12 +286,12 @@ class GatewayUsageLedger:
 
         owner_kind = raw.get("owner_kind") or "unknown"
         owner_id = raw.get("owner_id") or "unknown"
-        actor_kind = "guest" if owner_kind == "guest" else ("user" if owner_kind in ("user", "admin") else "system")
+        actor_kind, actor_uid, actor_role = audit_actor_for(owner_kind, owner_id)
         audit.record_system(
             "ai.gateway.call",
             actor_kind=actor_kind,
-            actor_uid=owner_id if actor_kind == "user" else None,
-            actor_role="admin" if owner_kind == "admin" else (actor_kind if actor_kind != "user" else "member"),
+            actor_uid=actor_uid,
+            actor_role=actor_role,
             origin="api",
             result="success" if raw.get("success") else "failure",
             summary=f"LLM call by {owner_kind}/{owner_id} (model {raw.get('model') or ''})",