retoor/devplacepy
1
0
Fork 1
Code Issues 1 Pull Requests 4 Actions Packages Projects Releases Wiki Activity

feat: add admin/internal database API with CRUD, read-only query, and natural-language SQL endpoints

Browse Source 
Add a new `/dbapi` router package providing a generic database API over `dataset`, restricted to admin sessions, admin API keys, and the internal gateway key. Includes:

- `tables.py`: list all tables and inspect table schemas
- `crud.py`: full CRUD operations (GET, POST, PATCH, DELETE) with soft-delete awareness, born-live inserts, `?include_deleted`, `.../restore`, and `?hard=true` purge
- `query.py`: validated read-only SELECT execution via sqlglot parsing, classification, and EXPLAIN dry-run; async query jobs with WebSocket streaming via `DbApiJobService`
- `nl.py`: natural-language-to-SQL conversion using the platform AI gateway with re-prompting until validation passes

Also register `DbApiJobService` and `PubSubService` in the service manager, add `DBAPI_DIR` to config data paths, and force cleartext `http://` connections to HTTP/1.1 in `curl_transport` to fix large request failures against uvicorn's HTTP/1.1-only internal gateway.
This commit is contained in:
retoor
2026-06-14 23:00:30 +00:00
parent 96521e8757
commit d31ccba6c1
69 changed files with 3278 additions and 12 deletions
Download Patch File Download Diff File Expand all files Collapse all files
devplacepy/config.py
+2
View File
@@ -25,6 +25,7 @@ ZIPS_DIR = DATA_DIR / "zips"
ZIP_STAGING_DIR = DATA_DIR / "zip_staging"
FORK_STAGING_DIR = DATA_DIR / "fork_staging"
SEO_REPORTS_DIR = DATA_DIR / "seo_reports"
DBAPI_DIR = DATA_DIR / "dbapi"
DEEPSEARCH_DIR = DATA_DIR / "deepsearch"
DEEPSEARCH_CHROMA_DIR = DEEPSEARCH_DIR / "chroma"
KEYS_DIR = DATA_DIR / "keys"
@@ -102,6 +103,7 @@ DATA_PATHS: dict[str, Path] = {
"zip_staging": ZIP_STAGING_DIR,
"fork_staging": FORK_STAGING_DIR,
"seo_reports": SEO_REPORTS_DIR,
"dbapi": DBAPI_DIR,
"deepsearch": DEEPSEARCH_DIR,
"deepsearch_chroma": DEEPSEARCH_CHROMA_DIR,
"keys": KEYS_DIR,
devplacepy/curl_transport.py
+11
View File
@@ -45,6 +45,12 @@ HTTP_VERSION_LABELS: dict[int, bytes] = {
}
def http_version_for(url: httpx.URL):
if url.scheme == "http":
return CurlHttpVersion.V1_1
return None
def resolve_timeout(request: httpx.Request) -> float:
extension = request.extensions.get("timeout") or {}
for key in ("read", "connect", "pool"):
@@ -79,6 +85,10 @@ class CurlTransport(httpx.AsyncBaseTransport):
if key.lower() not in STRIP_REQUEST_HEADERS
}
body = await request.aread()
extra = {}
version = http_version_for(request.url)
if version is not None:
extra["http_version"] = version
try:
response = await self._session.request(
request.method,
@@ -90,6 +100,7 @@ class CurlTransport(httpx.AsyncBaseTransport):
stream=True,
allow_redirects=False,
timeout=resolve_timeout(request),
**extra,
)
except Timeout as exc:
raise httpx.ConnectTimeout(str(exc), request=request) from exc
devplacepy/main.py
+8
View File
@@ -73,6 +73,8 @@ from devplacepy.routers import (
tools,
xmlrpc,
devrant,
dbapi,
pubsub,
)
from devplacepy.services.manager import service_manager
from devplacepy.services.background import background
@@ -84,6 +86,8 @@ from devplacepy.services.jobs.zip_service import ZipService
from devplacepy.services.jobs.fork_service import ForkService
from devplacepy.services.jobs.issue_create_service import IssueCreateService
from devplacepy.services.jobs.seo.service import SeoService
from devplacepy.services.dbapi.service import DbApiJobService
from devplacepy.services.pubsub import PubSubService
from devplacepy.services.jobs.deepsearch.service import DeepsearchService
from devplacepy.services.gitea.service import IssueTrackerService
from devplacepy.services.containers.service import ContainerService
@@ -331,6 +335,8 @@ app.include_router(proxy.router, prefix="/p")
app.include_router(tools.router, prefix="/tools")
app.include_router(xmlrpc.router, prefix="/xmlrpc")
app.include_router(devrant.router, prefix="/api")
app.include_router(dbapi.router, prefix="/dbapi")
app.include_router(pubsub.router, prefix="/pubsub")
@app.middleware("http")
@@ -449,6 +455,8 @@ async def startup():
service_manager.register(ZipService())
service_manager.register(ForkService())
service_manager.register(SeoService())
service_manager.register(DbApiJobService())
service_manager.register(PubSubService())
service_manager.register(DeepsearchService())
service_manager.register(IssueCreateService())
service_manager.register(IssueTrackerService())
devplacepy/routers/dbapi/__init__.py
+11
View File
@@ -0,0 +1,11 @@
# retoor <retoor@molodetz.nl>
from fastapi import APIRouter
from . import crud, nl, query, tables
router = APIRouter()
router.include_router(tables.router)
router.include_router(query.router)
router.include_router(nl.router)
router.include_router(crud.router)
devplacepy/routers/dbapi/_shared.py
+79
View File
@@ -0,0 +1,79 @@
# retoor <retoor@molodetz.nl>
from __future__ import annotations
from fastapi import HTTPException, Request
from fastapi.responses import JSONResponse
from devplacepy.services.dbapi import policy
from devplacepy.services.dbapi.policy import Caller, DbApiBadTable, DbApiDenied
OPERATORS = {"gte": ">=", "lte": "<=", "gt": ">", "lt": "<"}
RESERVED = {"limit", "before", "search", "include_deleted", "key", "hard", "execute"}
def require_dbapi_caller(request: Request) -> Caller:
try:
return policy.require_caller(request)
except DbApiDenied as exc:
from devplacepy.services.audit import record as audit
audit.record(
request,
"database.access.denied",
result="denied",
summary=f"denied database API access to {request.url.path}",
)
raise HTTPException(status_code=403, detail=str(exc))
def assert_table(name: str):
try:
return policy.assert_table(name)
except DbApiBadTable as exc:
raise HTTPException(status_code=404, detail=str(exc))
def error(status: int, message: str, **extra) -> JSONResponse:
return JSONResponse(
{"error": {"status": status, "message": message, **extra}}, status_code=status
)
async def read_body(request: Request) -> dict:
content_type = request.headers.get("content-type", "")
if content_type.startswith("application/json"):
try:
data = await request.json()
except Exception:
return {}
return data if isinstance(data, dict) else {}
form = await request.form()
return {key: value for key, value in form.items()}
def row_payload(body: dict) -> dict:
if "values_json" in body:
import json
try:
parsed = json.loads(body["values_json"])
except Exception:
return {}
return parsed if isinstance(parsed, dict) else {}
return {key: value for key, value in body.items() if key not in ("confirm", "values_json")}
def parse_filters(request: Request) -> tuple[dict, dict]:
filters: dict = {}
comparisons: dict = {}
for key, value in request.query_params.multi_items():
if key in RESERVED:
continue
if "." in key:
prefix, column = key.split(".", 1)
if prefix in ("filter", "eq"):
filters[column] = value
elif prefix in OPERATORS:
comparisons[column] = {OPERATORS[prefix]: value}
return filters, comparisons
devplacepy/routers/dbapi/crud.py
+152
View File
@@ -0,0 +1,152 @@
# retoor <retoor@molodetz.nl>
from fastapi import APIRouter, Request
from fastapi.responses import JSONResponse
from devplacepy.schemas import DbMutationOut, DbRowOut, DbRowsOut
from devplacepy.services.dbapi import crud
from devplacepy.services.dbapi.crud import DbApiError
from ._shared import (
assert_table,
error,
parse_filters,
read_body,
require_dbapi_caller,
row_payload,
)
router = APIRouter()
def _target_uid(row: dict, value: str) -> str:
return (row or {}).get("uid") or str(value)
@router.get("/{table}")
async def dbapi_list(
request: Request,
table: str,
limit: int = 25,
before: str = None,
search: str = "",
include_deleted: bool = False,
):
require_dbapi_caller(request)
assert_table(table)
filters, comparisons = parse_filters(request)
rows, next_cursor = crud.list_rows(
table,
filters=filters,
comparisons=comparisons,
search=search,
before=before,
limit=limit,
include_deleted=include_deleted,
)
return JSONResponse(
DbRowsOut(
table=table, rows=rows, count=len(rows), next_cursor=next_cursor
).model_dump(mode="json")
)
@router.get("/{table}/{key}/{value}")
async def dbapi_get(request: Request, table: str, key: str, value: str):
require_dbapi_caller(request)
assert_table(table)
try:
row = crud.get_row(table, key, value)
except DbApiError as exc:
return error(400, str(exc))
if row is None:
return error(404, "Row not found")
return JSONResponse(DbRowOut(table=table, row=row).model_dump(mode="json"))
@router.post("/{table}")
async def dbapi_insert(request: Request, table: str):
caller = require_dbapi_caller(request)
assert_table(table)
body = await read_body(request)
try:
row = crud.insert_row(table, row_payload(body), caller.uid)
except DbApiError as exc:
return error(400, str(exc))
_audit(request, "database.row.insert", table, _target_uid(row, ""), caller)
return JSONResponse(
DbMutationOut(table=table, ok=True, mode="insert", row=row).model_dump(mode="json")
)
@router.patch("/{table}/{key}/{value}")
async def dbapi_update(request: Request, table: str, key: str, value: str):
caller = require_dbapi_caller(request)
assert_table(table)
body = await read_body(request)
try:
row = crud.update_row(table, key, value, row_payload(body), caller.uid)
except DbApiError as exc:
return error(400, str(exc))
if row is None:
return error(404, "Row not found")
_audit(request, "database.row.update", table, _target_uid(row, value), caller)
return JSONResponse(
DbMutationOut(table=table, ok=True, mode="update", row=row).model_dump(mode="json")
)
@router.delete("/{table}/{key}/{value}")
async def dbapi_delete(
request: Request, table: str, key: str, value: str, hard: bool = False
):
caller = require_dbapi_caller(request)
assert_table(table)
try:
outcome = crud.delete_row(table, key, value, caller.uid, hard=hard)
except DbApiError as exc:
return error(400, str(exc))
if outcome is None:
return error(404, "Row not found")
_audit(
request,
"database.row.delete",
table,
_target_uid(outcome.get("row", {}), value),
caller,
metadata={"mode": outcome["mode"]},
)
return JSONResponse(
DbMutationOut(
table=table, ok=True, mode=outcome["mode"], row=outcome.get("row")
).model_dump(mode="json")
)
@router.post("/{table}/{key}/{value}/restore")
async def dbapi_restore(request: Request, table: str, key: str, value: str):
caller = require_dbapi_caller(request)
assert_table(table)
try:
row = crud.restore_row(table, key, value, caller.uid)
except DbApiError as exc:
return error(400, str(exc))
if row is None:
return error(404, "Row not found")
_audit(request, "database.row.restore", table, _target_uid(row, value), caller)
return JSONResponse(
DbMutationOut(table=table, ok=True, mode="restore", row=row).model_dump(mode="json")
)
def _audit(request, event_key, table, target_uid, caller, metadata=None):
from devplacepy.services.audit import record as audit
audit.record(
request,
event_key,
target_type=table,
target_uid=target_uid,
summary=f"{caller.username or caller.kind} {event_key} on {table}/{target_uid}",
metadata={"table": table, "caller": caller.kind, **(metadata or {})},
)
devplacepy/routers/dbapi/nl.py
+65
View File
@@ -0,0 +1,65 @@
# retoor <retoor@molodetz.nl>
from fastapi import APIRouter, Request
from fastapi.responses import JSONResponse
from devplacepy.database import get_int_setting
from devplacepy.schemas import NlQueryOut
from devplacepy.services.dbapi import nl2sql
from devplacepy.services.dbapi.validate import run_select
from ._shared import assert_table, error, read_body, require_dbapi_caller
router = APIRouter()
DEFAULT_MAX_ROWS = 5000
def _truthy(value) -> bool:
return str(value).strip().lower() in ("1", "true", "yes", "on")
@router.post("/nl")
async def dbapi_nl(request: Request):
caller = require_dbapi_caller(request)
body = await read_body(request)
question = str(body.get("question", "")).strip()
table = str(body.get("table", "")).strip()
if not question or not table:
return error(400, "Provide both 'question' and 'table'.")
assert_table(table)
apply_soft_delete = _truthy(body.get("apply_soft_delete", True))
dialect = str(body.get("dialect", "sqlite")).strip() or "sqlite"
execute = _truthy(body.get("execute", False))
design = await nl2sql.design_query(
question,
table,
apply_soft_delete=apply_soft_delete,
dialect=dialect,
api_key=caller.api_key,
)
out = NlQueryOut(**design.as_dict())
if execute and design.valid:
max_rows = max(1, get_int_setting("dbapi_max_rows", DEFAULT_MAX_ROWS))
rows, truncated = run_select(design.sql, max_rows)
out.executed = True
out.rows = rows
out.row_count = len(rows)
out.truncated = truncated
_audit_nl(request, caller, table, design.sql, design.valid)
return JSONResponse(out.model_dump(mode="json"))
def _audit_nl(request, caller, table, sql, valid):
from devplacepy.services.audit import record as audit
audit.record(
request,
"database.nl.design",
target_type=table,
summary=f"{caller.username or caller.kind} designed a query on {table}",
metadata={"table": table, "sql": sql[:500], "valid": valid, "caller": caller.kind},
)
devplacepy/routers/dbapi/query.py
+202
View File
@@ -0,0 +1,202 @@
# retoor <retoor@molodetz.nl>
import json
import logging
from fastapi import APIRouter, Request, WebSocket, WebSocketDisconnect
from fastapi.responses import JSONResponse
from devplacepy.config import DBAPI_DIR
from devplacepy.database import get_int_setting
from devplacepy.schemas import DbQueryJobOut, DbQueryOut
from devplacepy.services.dbapi import policy
from devplacepy.services.dbapi.progress import hub
from devplacepy.services.dbapi.validate import run_select, validate_select
from devplacepy.services.jobs import queue
from devplacepy.services.manager import service_manager
from ._shared import error, read_body, require_dbapi_caller
logger = logging.getLogger(__name__)
router = APIRouter()
TERMINAL_STATES = (queue.DONE, queue.FAILED)
DEFAULT_MAX_ROWS = 5000
def _owner(caller) -> tuple[str, str]:
if caller.kind == "admin":
return "user", caller.uid
return "internal", caller.uid
def _max_rows() -> int:
return max(1, get_int_setting("dbapi_max_rows", DEFAULT_MAX_ROWS))
@router.post("/query")
async def dbapi_query(request: Request):
caller = require_dbapi_caller(request)
body = await read_body(request)
sql = str(body.get("sql", "")).strip()
dialect = str(body.get("dialect", "sqlite")).strip() or "sqlite"
if not sql:
return error(400, "Provide a 'sql' SELECT statement.")
verdict = validate_select(sql, dialect=dialect)
if not verdict.valid:
status = 409 if not verdict.is_select else 400
hint = (
" Use the structured /dbapi/{table} CRUD routes to change data."
if not verdict.is_select
else ""
)
return JSONResponse(
DbQueryOut(
sql=verdict.sql,
valid=False,
statement_type=verdict.statement_type,
suspicious=verdict.suspicious,
error=(verdict.error or "Invalid query.") + hint,
).model_dump(mode="json"),
status_code=status,
)
rows, truncated = run_select(verdict.sql, _max_rows())
_audit_query(request, caller, verdict.sql, len(rows))
return JSONResponse(
DbQueryOut(
sql=verdict.sql,
valid=True,
statement_type="select",
rows=rows,
row_count=len(rows),
truncated=truncated,
suspicious=verdict.suspicious,
).model_dump(mode="json")
)
@router.post("/query/async")
async def dbapi_query_async(request: Request):
caller = require_dbapi_caller(request)
body = await read_body(request)
sql = str(body.get("sql", "")).strip()
if not sql:
return error(400, "Provide a 'sql' SELECT statement.")
verdict = validate_select(sql)
if not verdict.valid:
status = 409 if not verdict.is_select else 400
return JSONResponse(
DbQueryOut(
sql=verdict.sql,
valid=False,
statement_type=verdict.statement_type,
suspicious=verdict.suspicious,
error=verdict.error or "Invalid query.",
).model_dump(mode="json"),
status_code=status,
)
owner_kind, owner_id = _owner(caller)
uid = queue.enqueue(
"dbquery", {"sql": verdict.sql}, owner_kind, owner_id, "dbquery"
)
_audit_query(request, caller, verdict.sql, None, event="database.query.async")
return JSONResponse(
{
"uid": uid,
"status_url": f"/dbapi/query/{uid}",
"ws_url": f"/dbapi/query/{uid}/ws",
}
)
@router.get("/query/{uid}")
async def dbapi_query_status(request: Request, uid: str):
require_dbapi_caller(request)
job = queue.get_job(uid)
if not job or job.get("kind") != "dbquery":
return error(404, "Query job not found")
result = job.get("result", {})
done = job.get("status") == queue.DONE
return JSONResponse(
DbQueryJobOut(
uid=uid,
kind="dbquery",
status=job.get("status", ""),
ws_url=f"/dbapi/query/{uid}/ws",
result_url=f"/dbapi/query/{uid}/result" if done else None,
row_count=result.get("row_count") if done else None,
truncated=result.get("truncated") if done else None,
error=job.get("error") or None,
created_at=job.get("created_at"),
completed_at=job.get("completed_at") or None,
).model_dump(mode="json")
)
@router.get("/query/{uid}/result")
async def dbapi_query_result(request: Request, uid: str):
require_dbapi_caller(request)
job = queue.get_job(uid)
if not job or job.get("kind") != "dbquery":
return error(404, "Query job not found")
path = (DBAPI_DIR / uid / "result.json").resolve()
if not path.is_relative_to(DBAPI_DIR.resolve()) or not path.is_file():
return error(404, "Result not available")
queue.touch_job(uid, get_int_setting("dbquery_retention_seconds", 604800))
return JSONResponse(json.loads(path.read_text(encoding="utf-8")))
@router.websocket("/query/{uid}/ws")
async def dbapi_query_ws(websocket: WebSocket, uid: str):
await websocket.accept()
if not service_manager.owns_lock():
await websocket.close(code=4013)
return
if policy.caller_for(websocket) is None:
await websocket.close(code=1008)
return
job = queue.get_job(uid)
if not job or job.get("kind") != "dbquery":
await websocket.close(code=1008)
return
listener = hub.subscribe(uid)
try:
for frame in hub.snapshot(uid):
await websocket.send_json(frame)
current = queue.get_job(uid)
if current and current.get("status") in TERMINAL_STATES:
done = current.get("status") == queue.DONE
result = current.get("result", {})
await websocket.send_json(
{
"type": "done" if done else "failed",
"status": current.get("status"),
"row_count": result.get("row_count"),
"truncated": result.get("truncated"),
"result_url": f"/dbapi/query/{uid}/result" if done else None,
"error": current.get("error") or None,
}
)
return
while True:
frame = await listener.get()
await websocket.send_json(frame)
if frame.get("type") in ("done", "failed"):
break
except WebSocketDisconnect:
pass
except Exception:
logger.exception("dbquery websocket loop failed for %s", uid)
finally:
hub.unsubscribe(uid, listener)
def _audit_query(request, caller, sql, row_count, event="database.query"):
from devplacepy.services.audit import record as audit
audit.record(
request,
event,
summary=f"{caller.username or caller.kind} ran a database query",
metadata={"sql": sql[:500], "row_count": row_count, "caller": caller.kind},
)
devplacepy/routers/dbapi/tables.py
+36
View File
@@ -0,0 +1,36 @@
# retoor <retoor@molodetz.nl>
from fastapi import APIRouter, Request
from fastapi.responses import JSONResponse
from devplacepy.database import get_table
from devplacepy.schemas import DbSchemaOut, DbTableListOut
from devplacepy.services.dbapi import crud, policy
from ._shared import assert_table, require_dbapi_caller
router = APIRouter()
@router.get("/tables")
async def dbapi_tables(request: Request):
require_dbapi_caller(request)
tables = []
for name in policy.allowed_tables():
tables.append(
{
"name": name,
"row_count": get_table(name).count(),
"soft_delete": policy.soft_delete_aware(name),
}
)
return JSONResponse(
DbTableListOut(tables=tables, count=len(tables)).model_dump(mode="json")
)
@router.get("/{table}/schema")
async def dbapi_schema(request: Request, table: str):
require_dbapi_caller(request)
assert_table(table)
return JSONResponse(DbSchemaOut(**crud.schema(table)).model_dump(mode="json"))
devplacepy/routers/docs/pages.py
+14
View File
@@ -468,6 +468,20 @@ DOCS_PAGES = [
"admin": True,
"section": SECTION_SERVICES,
},
{
"slug": "services-dbapi",
"title": "Database API service",
"kind": "prose",
"admin": True,
"section": SECTION_SERVICES,
},
{
"slug": "services-pubsub",
"title": "Pub/Sub service",
"kind": "prose",
"admin": True,
"section": SECTION_SERVICES,
},
# Architecture - platform design, structure, and development process (admins only)
{
"slug": "architecture",
devplacepy/routers/issues/index.py
+2 -1
View File
@@ -99,6 +99,7 @@ async def issue_detail(request: Request, number: int):
uids = [uid for uid in [author_uid, *comment_authors.values()] if uid]
users_map = get_users_by_uids(uids)
body = issue.get("body", "")
issue = issue_item(issue, author_uid, users_map)
comment_items = [
comment_item(
@@ -119,7 +120,7 @@ async def issue_detail(request: Request, number: int):
"request": request,
"user": user,
"issue": issue,
"body": issue.get("body", ""),
"body": body,
"comments": comment_items,
"can_comment": user is not None,
"viewer_is_admin": is_admin(user),
devplacepy/routers/projects/containers/instances.py
+8
View File
@@ -284,6 +284,14 @@ async def instance_exec_ws(websocket: WebSocket, project_slug: str, uid: str):
):
await websocket.close(code=1011)
return
if inst.get("status") != "running":
await websocket.send_text(
"\r\n[devplace] This container is "
f"{inst.get('status') or 'not running'}. Start it from the container page, "
"then open the terminal.\r\n"
)
await websocket.close(code=1011)
return
master, slave = pty.openpty()
session = (websocket.query_params.get("session") or "").strip()
devplacepy/routers/pubsub.py
+115
View File
@@ -0,0 +1,115 @@
# retoor <retoor@molodetz.nl>
import json
import logging
from datetime import datetime, timezone
from fastapi import APIRouter, Request, WebSocket, WebSocketDisconnect
from fastapi.responses import JSONResponse
from devplacepy.services.manager import service_manager
from devplacepy.services.pubsub import policy
from devplacepy.services.pubsub.hub import pubsub
from devplacepy.routers.dbapi._shared import error, read_body, require_dbapi_caller
logger = logging.getLogger(__name__)
router = APIRouter()
def _now() -> str:
return datetime.now(timezone.utc).isoformat()
def _message(topic: str, data) -> dict:
return {"type": "message", "topic": topic, "data": data, "ts": _now()}
@router.websocket("/ws")
async def pubsub_ws(websocket: WebSocket):
await websocket.accept()
svc = service_manager.get_service("pubsub")
if svc is None or not svc.is_enabled():
await websocket.close(code=1013)
return
if not service_manager.owns_lock():
await websocket.close(code=4013)
return
actor = policy.resolve_actor(websocket)
if actor.kind == "guest" and not policy.guests_enabled():
await websocket.close(code=1008)
return
await websocket.send_json({"type": "ready", "actor": actor.kind})
try:
while True:
data = await websocket.receive_json()
kind = data.get("type")
topic = str(data.get("topic", "")).strip()
if kind in ("subscribe", "unsubscribe", "publish") and not policy.valid_topic(
topic
):
await websocket.send_json({"type": "error", "message": "Invalid topic name."})
continue
if kind == "subscribe":
if policy.can_subscribe(actor, topic):
pubsub.subscribe(topic, websocket)
await websocket.send_json({"type": "subscribed", "topic": topic})
else:
await websocket.send_json(
{"type": "error", "message": f"Not allowed to subscribe to {topic}."}
)
elif kind == "unsubscribe":
pubsub.unsubscribe(topic, websocket)
await websocket.send_json({"type": "unsubscribed", "topic": topic})
elif kind == "publish":
if not policy.can_publish(actor, topic):
await websocket.send_json(
{"type": "error", "message": f"Not allowed to publish to {topic}."}
)
continue
payload = data.get("data")
if len(json.dumps(payload, default=str)) > policy.MAX_PAYLOAD_BYTES:
await websocket.send_json({"type": "error", "message": "Payload too large."})
continue
delivered = await pubsub.publish(topic, _message(topic, payload))
await websocket.send_json(
{"type": "ack", "topic": topic, "delivered": delivered}
)
else:
await websocket.send_json({"type": "error", "message": "Unknown frame type."})
except WebSocketDisconnect:
pass
except Exception:
logger.exception("pubsub websocket loop failed")
finally:
pubsub.drop_socket(websocket)
@router.post("/publish")
async def pubsub_http_publish(request: Request):
caller = require_dbapi_caller(request)
if not service_manager.owns_lock():
return error(409, "Pub/sub is served by the lock-owner worker. Retry.")
body = await read_body(request)
topic = str(body.get("topic", "")).strip()
if not policy.valid_topic(topic):
return error(400, "Invalid topic name.")
payload = body.get("data")
delivered = await pubsub.publish(topic, _message(topic, payload))
from devplacepy.services.audit import record as audit
audit.record(
request,
"pubsub.publish",
target_type="topic",
target_uid=topic,
summary=f"{caller.username or caller.kind} published to {topic}",
metadata={"topic": topic, "delivered": delivered, "caller": caller.kind},
)
return JSONResponse({"topic": topic, "delivered": delivered})
@router.get("/topics")
async def pubsub_list_topics(request: Request):
require_dbapi_caller(request)
return JSONResponse({"topics": pubsub.topics()})
devplacepy/schemas.py
+86
View File
@@ -936,3 +936,89 @@ class AuditLogOut(_Out):
class AuditEventOut(_Out):
event: Optional[AuditEntryOut] = None
links: list[AuditLinkOut] = []
# ---------- database API ----------
class DbTableOut(_Out):
name: str = ""
row_count: int = 0
soft_delete: bool = False
class DbTableListOut(_Out):
tables: list[DbTableOut] = []
count: int = 0
class DbColumnOut(_Out):
name: str = ""
type: str = ""
class DbSchemaOut(_Out):
table: str = ""
columns: list[DbColumnOut] = []
soft_delete: bool = False
row_count: int = 0
class DbRowsOut(_Out):
table: str = ""
rows: list[dict] = []
count: int = 0
next_cursor: Optional[Any] = None
class DbRowOut(_Out):
table: str = ""
row: Optional[dict] = None
class DbMutationOut(_Out):
table: str = ""
ok: bool = True
mode: Optional[str] = None
row: Optional[dict] = None
class DbQueryOut(_Out):
sql: str = ""
valid: bool = False
statement_type: str = ""
rows: list[dict] = []
row_count: int = 0
truncated: bool = False
suspicious: list[str] = []
error: Optional[str] = None
class DbQueryJobOut(_Out):
uid: str = ""
kind: str = ""
status: str = ""
ws_url: Optional[str] = None
result_url: Optional[str] = None
row_count: Optional[int] = None
truncated: Optional[bool] = None
error: Optional[str] = None
created_at: Optional[str] = None
completed_at: Optional[str] = None
class NlQueryOut(_Out):
question: str = ""
table: str = ""
sql: str = ""
valid: bool = False
attempts: int = 0
dialect: str = "sqlite"
applied_soft_delete: bool = False
suspicious: list[str] = []
tables: list[str] = []
executed: bool = False
rows: list[dict] = []
row_count: int = 0
truncated: bool = False
error: Optional[str] = None
devplacepy/services/audit/categories.py
+2
View File
@@ -29,6 +29,8 @@ CATEGORY_BY_PREFIX: dict[str, str] = {
"seo": "tools",
"deepsearch": "tools",
"ai": "ai",
"database": "database",
"pubsub": "pubsub",
"devii": "devii",
"cli": "cli",
"reward": "reward",
devplacepy/services/containers/files/bot.py
+8 -4
View File
@@ -2537,13 +2537,14 @@ async def handle_mentions(dp: DevPlace, answered: set[str]) -> None:
reply = await _agent_answer_for_devplace(
f"You were @-mentioned in a DevPlace post. The message to you is:\n\n{message}\n\n"
f"Do what it asks, then answer it directly. Your reply is posted verbatim as a "
f"comment, so write it as botje talking to the user — not as a report about what you did.",
f"comment, so write it as botje talking to the user — not as a report about what you did. "
f"Keep the entire reply within 1000 characters (the comment length limit).",
context=f"Post URL: {DEVPLACE_URL}/posts/{slug}",
)
dp.call(
"comments.create",
content=reply[:5000],
content=reply.strip()[:1000],
target_uid=post_uid,
target_type="post",
)
@@ -2597,10 +2598,13 @@ async def handle_dms(dp: DevPlace, answered: set[str]) -> None:
try:
reply = await _agent_answer_for_devplace(
content,
context=f"The user @{other_name} (uid {other_uid}) sent you a direct message.",
context=(
f"The user @{other_name} (uid {other_uid}) sent you a direct message. "
f"Keep the entire reply within 2000 characters (the message length limit)."
),
)
dp.call("messages.send", content=reply[:5000], receiver_uid=other_uid)
dp.call("messages.send", content=reply.strip()[:2000], receiver_uid=other_uid)
logger.info("Replied to DM from @%s", other_name)
except (xmlrpc.client.Fault, Exception) as e:
devplacepy/services/containers/service.py
+4 -1
View File
@@ -170,8 +170,11 @@ class ContainerService(BaseService):
elif ps.state == "created":
await backend.start(ps.container_id)
_set_status(inst, {"status": store.ST_RUNNING}, reason="start")
else:
elif status == store.ST_RUNNING:
await self._handle_exit(backend, inst, ps)
else:
await backend.rm(ps.container_id, force=True)
await self._launch(backend, inst)
elif desired == store.DESIRED_STOPPED:
if ps is not None and ps.state in ("running", "restarting", "paused"):
devplacepy/services/dbapi/__init__.py
+1
View File
@@ -0,0 +1 @@
# retoor <retoor@molodetz.nl>
devplacepy/services/dbapi/crud.py
+176
View File
@@ -0,0 +1,176 @@
# retoor <retoor@molodetz.nl>
from __future__ import annotations
from datetime import datetime, timezone
from devplacepy.database import (
get_table,
purge,
restore,
soft_delete,
text_search_clause,
)
from devplacepy.utils import generate_uid
from .policy import soft_delete_aware
SEARCH_FIELDS = (
"title",
"description",
"content",
"name",
"username",
"email",
"body",
"summary",
"slug",
)
MAX_SYNC_LIMIT = 500
class DbApiError(Exception):
pass
def _now() -> str:
return datetime.now(timezone.utc).isoformat()
def column_names(table_name: str) -> list[str]:
return list(get_table(table_name).columns)
def schema(table_name: str) -> dict:
table = get_table(table_name)
columns = []
for column in table.table.columns:
columns.append({"name": column.name, "type": str(column.type)})
return {
"table": table_name,
"columns": columns,
"soft_delete": soft_delete_aware(table_name),
"row_count": table.count(),
}
def _cursor_field(table) -> str:
return "created_at" if table.has_column("created_at") else "id"
def list_rows(
table_name: str,
*,
filters: dict | None = None,
comparisons: dict | None = None,
search: str = "",
before=None,
limit: int = 25,
include_deleted: bool = False,
) -> tuple[list[dict], object]:
table = get_table(table_name)
cols = set(column_names(table_name))
limit = max(1, min(int(limit or 25), MAX_SYNC_LIMIT))
cursor_field = _cursor_field(table)
clauses = []
column_objs = table.table.columns
if soft_delete_aware(table_name) and "deleted_at" in cols and not include_deleted:
clauses.append(column_objs.deleted_at.is_(None))
fields = tuple(field for field in SEARCH_FIELDS if field in cols)
search_clause = text_search_clause(table, search, fields=fields) if fields else None
if search_clause is not None:
clauses.append(search_clause)
if before is not None:
clauses.append(column_objs[cursor_field] < before)
kwargs = {}
for key, value in (filters or {}).items():
if key in cols:
kwargs[key] = value
for key, expression in (comparisons or {}).items():
if key in cols:
kwargs[key] = expression
rows = list(
table.find(*clauses, order_by=["-" + cursor_field], _limit=limit + 1, **kwargs)
)
has_more = len(rows) > limit
rows = rows[:limit]
next_cursor = rows[-1][cursor_field] if has_more and rows else None
return [dict(row) for row in rows], next_cursor
def count_rows(table_name: str, filters: dict | None = None) -> int:
table = get_table(table_name)
cols = set(column_names(table_name))
kwargs = {key: value for key, value in (filters or {}).items() if key in cols}
return table.count(**kwargs)
def get_row(table_name: str, key: str, value) -> dict | None:
_assert_key(table_name, key)
row = get_table(table_name).find_one(**{key: value})
return dict(row) if row else None
def insert_row(table_name: str, data: dict, actor: str) -> dict:
table = get_table(table_name)
cols = set(column_names(table_name))
row = dict(data or {})
unknown = [key for key in row if key not in cols]
if unknown:
raise DbApiError(f"Unknown columns for {table_name}: {', '.join(sorted(unknown))}")
if "uid" in cols and not row.get("uid"):
row["uid"] = generate_uid()
if "created_at" in cols and not row.get("created_at"):
row["created_at"] = _now()
if soft_delete_aware(table_name):
row.setdefault("deleted_at", None)
row.setdefault("deleted_by", None)
table.insert(row)
if row.get("uid"):
return get_row(table_name, "uid", row["uid"]) or row
return row
def update_row(table_name: str, key: str, value, data: dict, actor: str) -> dict | None:
_assert_key(table_name, key)
table = get_table(table_name)
cols = set(column_names(table_name))
if not table.find_one(**{key: value}):
return None
payload = {k: v for k, v in (data or {}).items() if k not in ("id", "uid", key)}
unknown = [k for k in payload if k not in cols]
if unknown:
raise DbApiError(f"Unknown columns for {table_name}: {', '.join(sorted(unknown))}")
if "updated_at" in cols:
payload["updated_at"] = _now()
payload[key] = value
table.update(payload, [key])
return get_row(table_name, key, value)
def delete_row(
table_name: str, key: str, value, actor: str, *, hard: bool = False
) -> dict | None:
_assert_key(table_name, key)
table = get_table(table_name)
row = table.find_one(**{key: value})
if not row:
return None
if soft_delete_aware(table_name) and not hard:
soft_delete(table_name, actor, **{key: value})
return {"mode": "soft", "row": dict(row)}
purge(table_name, **{key: value})
return {"mode": "hard", "row": dict(row)}
def restore_row(table_name: str, key: str, value, actor: str) -> dict | None:
_assert_key(table_name, key)
if not soft_delete_aware(table_name):
raise DbApiError(f"{table_name} does not support restore (no soft delete).")
restore(table_name, **{key: value})
return get_row(table_name, key, value)
def _assert_key(table_name: str, key: str) -> None:
if key not in set(column_names(table_name)):
raise DbApiError(f"Unknown key column {key!r} for {table_name}.")
devplacepy/services/dbapi/nl2sql.py
+148
View File
@@ -0,0 +1,148 @@
# retoor <retoor@molodetz.nl>
from __future__ import annotations
import json
import logging
import re
from dataclasses import dataclass, field
from devplacepy import stealth
from devplacepy.config import INTERNAL_GATEWAY_URL, INTERNAL_MODEL
from devplacepy.database import get_setting, internal_gateway_key
from . import crud
from .policy import soft_delete_aware
from .validate import validate_select
logger = logging.getLogger(__name__)
GATEWAY_TIMEOUT_SECONDS = 60.0
MAX_TOKENS = 600
FENCE = re.compile(r"```(?:sql)?\s*(.+?)```", re.IGNORECASE | re.DOTALL)
@dataclass
class Design:
question: str
table: str
sql: str = ""
valid: bool = False
attempts: int = 0
dialect: str = "sqlite"
applied_soft_delete: bool = False
suspicious: list[str] = field(default_factory=list)
tables: list[str] = field(default_factory=list)
error: str | None = None
def as_dict(self) -> dict:
return {
"question": self.question,
"table": self.table,
"sql": self.sql,
"valid": self.valid,
"attempts": self.attempts,
"dialect": self.dialect,
"applied_soft_delete": self.applied_soft_delete,
"suspicious": self.suspicious,
"tables": self.tables,
"error": self.error,
}
def _extract_sql(text: str) -> str:
match = FENCE.search(text or "")
sql = match.group(1) if match else (text or "")
return sql.strip().rstrip(";").strip()
def _system_prompt(table: str, apply_soft_delete: bool, dialect: str) -> str:
schema = crud.schema(table)
columns = ", ".join(f"{c['name']} ({c['type']})" for c in schema["columns"])
examples, _ = crud.list_rows(table, limit=5)
sample = json.dumps(examples, ensure_ascii=False, default=str)[:3000]
preamble = get_setting("dbapi_nl_system_preamble", "").strip()
soft_rule = ""
if apply_soft_delete and soft_delete_aware(table):
soft_rule = (
f"The {table} table uses soft deletes. ALWAYS add `deleted_at IS NULL` to the "
"WHERE clause so deleted rows are excluded, UNLESS the user explicitly asks for "
"deleted or all rows.\n"
)
lines = [
preamble,
"You translate a natural-language request into ONE read-only SQL SELECT statement "
f"for a SQLite database (dialect: {dialect}).",
f"Target table: {table}",
f"Columns: {columns}",
f"Example rows (JSON): {sample}",
soft_rule,
"Rules: return ONLY the SQL, no prose, no explanation, no markdown. Exactly one "
"SELECT statement. Never write INSERT, UPDATE, DELETE, DROP, PRAGMA, or ATTACH. "
"Prefer an explicit LIMIT when the user does not ask for everything. Dates are ISO "
"8601 strings; compare them lexicographically.",
]
return "\n".join(line for line in lines if line)
async def _complete(messages: list[dict], api_key: str, model: str) -> str:
payload = {
"model": model,
"messages": messages,
"max_tokens": MAX_TOKENS,
"temperature": 0.0,
}
headers = {"Authorization": f"Bearer {api_key}", "Content-Type": "application/json"}
async with stealth.stealth_async_client(timeout=GATEWAY_TIMEOUT_SECONDS) as client:
response = await client.post(INTERNAL_GATEWAY_URL, json=payload, headers=headers)
if response.status_code >= 400:
raise RuntimeError(f"AI gateway returned {response.status_code}")
data = response.json()
return (data.get("choices") or [{}])[0].get("message", {}).get("content") or ""
async def design_query(
question: str,
table: str,
*,
apply_soft_delete: bool = True,
dialect: str = "sqlite",
max_attempts: int = 3,
api_key: str = "",
) -> Design:
design = Design(question=question, table=table, dialect=dialect)
key = api_key or internal_gateway_key()
model = get_setting("dbapi_nl_model", "") or INTERNAL_MODEL
messages = [
{"role": "system", "content": _system_prompt(table, apply_soft_delete, dialect)},
{"role": "user", "content": question},
]
for attempt in range(1, max(1, max_attempts) + 1):
design.attempts = attempt
try:
raw = await _complete(messages, key, model)
except Exception as exc:
design.error = f"AI gateway error: {exc}"
return design
sql = _extract_sql(raw)
verdict = validate_select(sql, dialect=dialect)
design.sql = verdict.sql
design.suspicious = verdict.suspicious
design.tables = verdict.tables
design.applied_soft_delete = "deleted_at" in verdict.sql.lower()
if verdict.valid:
design.valid = True
design.error = None
return design
design.error = verdict.error
messages.append({"role": "assistant", "content": sql})
messages.append(
{
"role": "user",
"content": (
f"That query is invalid: {verdict.error}. Return a corrected single "
"SELECT statement only, no prose."
),
}
)
return design
devplacepy/services/dbapi/policy.py
+103
View File
@@ -0,0 +1,103 @@
# retoor <retoor@molodetz.nl>
from __future__ import annotations
import re
from dataclasses import dataclass
from devplacepy.database import (
SOFT_DELETE_TABLES,
db,
get_setting,
internal_gateway_key,
)
from devplacepy.utils import _user_from_api_key, get_current_user, is_admin
TABLE_NAME = re.compile(r"^[A-Za-z_][A-Za-z0-9_]*$")
DEFAULT_DENY = {"sessions", "password_resets", "cache_state"}
class DbApiDenied(Exception):
pass
class DbApiBadTable(Exception):
pass
@dataclass(frozen=True)
class Caller:
kind: str
uid: str
username: str
api_key: str
@property
def is_internal(self) -> bool:
return self.kind == "internal"
def deny_tables() -> set[str]:
raw = get_setting("dbapi_deny_tables", "")
extra = {name.strip() for name in raw.split(",") if name.strip()}
return DEFAULT_DENY | extra
def _bearer_key(request) -> str:
key = request.headers.get("x-api-key", "").strip()
if key:
return key
scheme, _, credentials = request.headers.get("authorization", "").partition(" ")
if scheme.lower() == "bearer":
return credentials.strip()
return ""
def caller_for(request) -> Caller | None:
user = get_current_user(request)
if user and is_admin(user):
return Caller(
kind="admin",
uid=user["uid"],
username=user.get("username", ""),
api_key=user.get("api_key", "") or "",
)
key = _bearer_key(request)
if key and key == internal_gateway_key():
return Caller(kind="internal", uid="internal", username="internal", api_key=key)
if key:
keyed = _user_from_api_key(key)
if keyed and is_admin(keyed):
return Caller(
kind="admin",
uid=keyed["uid"],
username=keyed.get("username", ""),
api_key=keyed.get("api_key", "") or "",
)
return None
def require_caller(request) -> Caller:
caller = caller_for(request)
if caller is None:
raise DbApiDenied("Administrator or internal-key access required.")
return caller
def soft_delete_aware(table: str) -> bool:
return table in SOFT_DELETE_TABLES
def assert_table(name: str) -> str:
if not name or not TABLE_NAME.match(name):
raise DbApiBadTable(f"Invalid table name: {name!r}")
if name in deny_tables():
raise DbApiBadTable(f"Table {name!r} is not accessible through the database API.")
if name not in db.tables:
raise DbApiBadTable(f"Unknown table: {name!r}")
return name
def allowed_tables() -> list[str]:
blocked = deny_tables()
return sorted(name for name in db.tables if name not in blocked)
devplacepy/services/dbapi/progress.py
+43
View File
@@ -0,0 +1,43 @@
# retoor <retoor@molodetz.nl>
from __future__ import annotations
import asyncio
MAX_BUFFER = 2000
class ProgressHub:
def __init__(self) -> None:
self._subscribers: dict[str, set[asyncio.Queue]] = {}
self._buffers: dict[str, list[dict]] = {}
def subscribe(self, uid: str) -> asyncio.Queue:
queue: asyncio.Queue = asyncio.Queue()
self._subscribers.setdefault(uid, set()).add(queue)
return queue
def unsubscribe(self, uid: str, queue: asyncio.Queue) -> None:
listeners = self._subscribers.get(uid)
if not listeners:
return
listeners.discard(queue)
if not listeners:
self._subscribers.pop(uid, None)
def publish(self, uid: str, frame: dict) -> None:
buffer = self._buffers.setdefault(uid, [])
buffer.append(frame)
if len(buffer) > MAX_BUFFER:
del buffer[: len(buffer) - MAX_BUFFER]
for queue in self._subscribers.get(uid, set()):
queue.put_nowait(frame)
def snapshot(self, uid: str) -> list[dict]:
return list(self._buffers.get(uid, []))
def clear(self, uid: str) -> None:
self._buffers.pop(uid, None)
hub = ProgressHub()
devplacepy/services/dbapi/service.py
+153
View File
@@ -0,0 +1,153 @@
# retoor <retoor@molodetz.nl>
from __future__ import annotations
import asyncio
import json
import logging
import shutil
import sqlite3
from pathlib import Path
from devplacepy.config import DBAPI_DIR
from devplacepy.database import get_int_setting
from devplacepy.services.base import ConfigField
from devplacepy.services.jobs.base import JobService
from .progress import hub
from .validate import _database_path, validate_select
logger = logging.getLogger(__name__)
BATCH_SIZE = 200
DEFAULT_MAX_ROWS = 5000
class DbApiJobService(JobService):
kind = "dbquery"
title = "Database API"
description = (
"Runs validated read-only SQL SELECT queries off the request path and streams the "
"result rows over a websocket. Backs the asynchronous /dbapi/query/async endpoint."
)
def __init__(self):
super().__init__(name="dbquery", interval_seconds=2)
self.config_fields.extend(
[
ConfigField(
"dbapi_max_rows",
"Max result rows",
type="int",
default=DEFAULT_MAX_ROWS,
minimum=1,
help="Hard cap on rows returned by a query (sync and async).",
group="Database API",
),
ConfigField(
"dbapi_nl_model",
"NL-to-SQL model",
type="str",
default="",
help="Model used to design SQL from natural language. Blank uses molodetz.",
group="Database API",
),
ConfigField(
"dbapi_nl_system_preamble",
"NL-to-SQL preamble",
type="text",
default="",
help="Optional operator text prepended to the NL-to-SQL system prompt.",
group="Database API",
),
ConfigField(
"dbapi_deny_tables",
"Denied tables",
type="str",
default="",
help="Comma separated extra tables to hide from the database API.",
group="Database API",
),
]
)
def result_dir(self, uid: str) -> Path:
return DBAPI_DIR / uid
async def process(self, job: dict) -> dict:
uid = job["uid"]
payload = job.get("payload", {})
sql = payload.get("sql", "")
verdict = validate_select(sql)
if not verdict.valid:
hub.publish(uid, {"type": "failed", "message": verdict.error or "invalid query"})
hub.clear(uid)
raise RuntimeError(verdict.error or "invalid query")
max_rows = max(1, get_int_setting("dbapi_max_rows", DEFAULT_MAX_ROWS))
rows, truncated = await self._stream(uid, verdict.sql, max_rows)
output_dir = self.result_dir(uid)
output_dir.mkdir(parents=True, exist_ok=True)
result = {
"sql": verdict.sql,
"row_count": len(rows),
"truncated": truncated,
"suspicious": verdict.suspicious,
"rows": rows,
}
(output_dir / "result.json").write_text(
json.dumps(result, default=str), encoding="utf-8"
)
hub.publish(
uid,
{
"type": "done",
"row_count": len(rows),
"truncated": truncated,
"result_url": f"/dbapi/query/{uid}/result",
},
)
hub.clear(uid)
return {
"sql": verdict.sql,
"row_count": len(rows),
"truncated": truncated,
"suspicious": verdict.suspicious,
"result_url": f"/dbapi/query/{uid}/result",
"bytes_out": len(json.dumps(result, default=str)),
"item_count": len(rows),
}
async def _stream(self, uid: str, sql: str, max_rows: int):
connection = sqlite3.connect(
f"file:{_database_path()}?mode=ro", uri=True, timeout=30
)
rows: list[dict] = []
truncated = False
try:
connection.row_factory = sqlite3.Row
connection.execute("PRAGMA query_only=ON")
cursor = connection.execute(sql)
while True:
batch = cursor.fetchmany(BATCH_SIZE)
if not batch:
break
for raw in batch:
if len(rows) >= max_rows:
truncated = True
break
rows.append(dict(raw))
hub.publish(
uid, {"type": "progress", "row_count": len(rows)}
)
if truncated:
break
await asyncio.sleep(0)
finally:
connection.close()
return rows, truncated
def cleanup(self, job: dict) -> None:
hub.clear(job["uid"])
shutil.rmtree(self.result_dir(job["uid"]), ignore_errors=True)
devplacepy/services/dbapi/validate.py
+156
View File
@@ -0,0 +1,156 @@
# retoor <retoor@molodetz.nl>
from __future__ import annotations
import re
import sqlite3
from dataclasses import dataclass, field
import sqlglot
from sqlglot import exp
from devplacepy.config import DATABASE_URL
UNSAFE = re.compile(r"\b(attach|detach|pragma|vacuum|reindex)\b", re.IGNORECASE)
_STATEMENT_TYPES = {
exp.Select: "select",
exp.Union: "select",
exp.Insert: "insert",
exp.Update: "update",
exp.Delete: "delete",
exp.Create: "ddl",
exp.Drop: "ddl",
exp.Alter: "ddl",
}
@dataclass
class Verdict:
sql: str
statement_type: str = "other"
is_select: bool = False
valid: bool = False
tables: list[str] = field(default_factory=list)
has_where: bool = False
has_join: bool = False
has_limit: bool = False
suspicious: list[str] = field(default_factory=list)
error: str | None = None
def as_dict(self) -> dict:
return {
"sql": self.sql,
"statement_type": self.statement_type,
"is_select": self.is_select,
"valid": self.valid,
"tables": self.tables,
"has_where": self.has_where,
"has_join": self.has_join,
"has_limit": self.has_limit,
"suspicious": self.suspicious,
"error": self.error,
}
def _database_path() -> str:
return DATABASE_URL.split("sqlite:///", 1)[-1]
def _statement_type(node: exp.Expression) -> str:
for kind, label in _STATEMENT_TYPES.items():
if isinstance(node, kind):
return label
return "other"
def classify(sql: str, dialect: str = "sqlite") -> Verdict:
text = (sql or "").strip().rstrip(";").strip()
verdict = Verdict(sql=text)
if not text:
verdict.error = "Empty query."
return verdict
try:
statements = [node for node in sqlglot.parse(text, read=dialect) if node]
except Exception as exc:
verdict.error = f"Could not parse SQL: {str(exc).splitlines()[0][:200]}"
return verdict
if not statements:
verdict.error = "No statement found."
return verdict
node = statements[0]
verdict.statement_type = _statement_type(node)
verdict.is_select = verdict.statement_type == "select"
verdict.tables = sorted({t.name for t in node.find_all(exp.Table) if t.name})
verdict.has_where = node.find(exp.Where) is not None
verdict.has_join = node.find(exp.Join) is not None
verdict.has_limit = node.find(exp.Limit) is not None
if len(statements) > 1:
verdict.suspicious.append("Multiple statements in one query are not allowed.")
if UNSAFE.search(text):
verdict.suspicious.append("ATTACH/PRAGMA/VACUUM style statements are not allowed.")
verdict.is_select = False
if verdict.is_select and not (
verdict.has_where or verdict.has_join or verdict.has_limit
):
verdict.suspicious.append(
"SELECT has no WHERE, JOIN, or LIMIT and may return an entire table."
)
return verdict
def dry_run(sql: str) -> tuple[bool, str | None]:
text = (sql or "").strip().rstrip(";").strip()
connection = sqlite3.connect(f"file:{_database_path()}?mode=ro", uri=True, timeout=5)
try:
connection.execute("PRAGMA query_only=ON")
connection.execute(f"EXPLAIN {text}")
return True, None
except sqlite3.Error as exc:
return False, str(exc)[:300]
finally:
connection.close()
def validate_select(sql: str, dialect: str = "sqlite") -> Verdict:
verdict = classify(sql, dialect=dialect)
if verdict.error:
return verdict
if len(verdict.suspicious) and any(
"Multiple statements" in note or "not allowed" in note
for note in verdict.suspicious
):
verdict.error = verdict.suspicious[0]
return verdict
if not verdict.is_select:
verdict.error = (
f"Only SELECT queries run through query(); this is a {verdict.statement_type} "
"statement. Use the structured /dbapi/{table} CRUD routes to change data."
)
return verdict
ok, error = dry_run(verdict.sql)
verdict.valid = ok
if not ok:
verdict.error = error
return verdict
def run_select(sql: str, limit: int) -> tuple[list[dict], bool]:
text = sql.strip().rstrip(";").strip()
connection = sqlite3.connect(f"file:{_database_path()}?mode=ro", uri=True, timeout=30)
try:
connection.row_factory = sqlite3.Row
connection.execute("PRAGMA query_only=ON")
cursor = connection.execute(text)
rows = [dict(row) for row in cursor.fetchmany(limit + 1)]
truncated = len(rows) > limit
return rows[:limit], truncated
finally:
connection.close()
def transpile(sql: str, read: str = "sqlite", write: str = "sqlite") -> str:
try:
return sqlglot.transpile(sql, read=read, write=write)[0]
except Exception:
return sql
devplacepy/services/devii/actions/catalog.py
+169
View File
@@ -1400,6 +1400,175 @@ ACTIONS: tuple[Action, ...] = (
params=(path("uid", "Attachment uid to restore."),),
requires_admin=True,
),
Action(
name="db_list_tables",
method="GET",
path="/dbapi/tables",
summary="List database tables exposed by the database API (admin only)",
description=(
"Returns every table reachable through the database API with its row count and "
"whether it uses soft deletes. Use this to discover what data exists before "
"querying or designing a query."
),
params=(),
requires_admin=True,
),
Action(
name="db_table_schema",
method="GET",
path="/dbapi/{table}/schema",
summary="Show a table's columns and types (admin only)",
description="Returns the column names, types, row count, and soft-delete flag for one table.",
params=(path("table", "Table name (from db_list_tables)."),),
requires_admin=True,
),
Action(
name="db_list_rows",
method="GET",
path="/dbapi/{table}",
summary="List rows of a table with keyset pagination (admin only)",
description=(
"Browses rows newest-first. Soft-deleted rows are excluded unless include_deleted is "
"true. For filtered or joined questions prefer db_query or db_design_query."
),
params=(
path("table", "Table name."),
query("limit", "Maximum rows (1-500, default 25)."),
query("search", "Free-text search over common text columns."),
query("before", "Keyset cursor: return rows older than this created_at/id value."),
Param(
name="include_deleted",
location="query",
description="Include soft-deleted rows.",
required=False,
type="boolean",
),
),
requires_admin=True,
),
Action(
name="db_get_row",
method="GET",
path="/dbapi/{table}/{key}/{value}",
summary="Fetch one row by a key column (admin only)",
description="Returns a single row where key column equals value (key is usually 'uid').",
params=(
path("table", "Table name."),
path("key", "Key column to match (usually 'uid')."),
path("value", "Value of the key column."),
),
requires_admin=True,
),
Action(
name="db_query",
method="POST",
path="/dbapi/query",
summary="Run a read-only SQL SELECT and return rows (admin only)",
description=(
"Executes a SINGLE validated SELECT statement read-only and returns the rows. Only "
"SELECT is allowed; INSERT/UPDATE/DELETE/DDL are rejected (use db_insert_row, "
"db_update_row, db_delete_row for changes). The response may include a 'suspicious' "
"list (e.g. a SELECT with no WHERE/JOIN/LIMIT that scans a whole table); when present, "
"surface that warning to the user before trusting the results."
),
params=(
body("sql", "A single SELECT statement.", required=True),
body("dialect", "Optional source SQL dialect (default sqlite)."),
),
requires_admin=True,
read_only=True,
),
Action(
name="db_design_query",
method="POST",
path="/dbapi/nl",
summary="Design a SQL SELECT from a natural-language question (admin only)",
description=(
"Turns a plain-language question about one table into a validated read-only SELECT. "
"It auto-adds 'deleted_at IS NULL' for soft-delete tables unless apply_soft_delete is "
"false. By default it only returns the SQL; pass execute=true to also run it read-only "
"and return rows. Show the user the SQL and any 'suspicious' notes."
),
params=(
body("question", "The natural-language request.", required=True),
body("table", "Target table the question is about.", required=True),
Param(
name="apply_soft_delete",
location="body",
description="Add deleted_at IS NULL for soft-delete tables (default true).",
required=False,
type="boolean",
),
body("dialect", "Optional target SQL dialect (default sqlite)."),
Param(
name="execute",
location="body",
description="Also run the validated query read-only and return rows.",
required=False,
type="boolean",
),
),
requires_admin=True,
read_only=True,
),
Action(
name="db_insert_row",
method="POST",
path="/dbapi/{table}",
summary="Insert a row into a table (admin only, confirmation required)",
description=(
"Inserts a new row. Pass the column values as a JSON object string in values_json. "
"Soft-delete columns and uid/created_at are filled automatically. Requires confirmation."
),
params=(
path("table", "Table name."),
body("values_json", "JSON object of column:value pairs for the new row.", required=True),
confirm(),
),
requires_admin=True,
),
Action(
name="db_update_row",
method="PATCH",
path="/dbapi/{table}/{key}/{value}",
summary="Update a row in a table (admin only, confirmation required)",
description=(
"Updates the row where key equals value. Pass the changed columns as a JSON object "
"string in values_json. uid and id cannot be changed. Requires confirmation."
),
params=(
path("table", "Table name."),
path("key", "Key column to match (usually 'uid')."),
path("value", "Value of the key column."),
body("values_json", "JSON object of column:value pairs to change.", required=True),
confirm(),
),
requires_admin=True,
),
Action(
name="db_delete_row",
method="DELETE",
path="/dbapi/{table}/{key}/{value}",
summary="Delete a row from a table (admin only, confirmation required)",
description=(
"Soft-deletes the row where key equals value (restorable). Pass hard=true to "
"PERMANENTLY purge it (or for tables without soft delete). Requires confirmation."
),
params=(
path("table", "Table name."),
path("key", "Key column to match (usually 'uid')."),
path("value", "Value of the key column."),
Param(
name="hard",
location="query",
description="Permanently purge instead of soft delete.",
required=False,
type="boolean",
),
confirm(),
),
requires_admin=True,
),
)
PLATFORM_CATALOG = Catalog(actions=ACTIONS)
devplacepy/services/devii/actions/dispatcher.py
+24 -1
View File
@@ -51,6 +51,9 @@ CONFIRM_REQUIRED = {
"admin_reset_guest_ai_quota",
"admin_reset_user_ai_quota",
"notification_reset",
"db_insert_row",
"db_update_row",
"db_delete_row",
}
CONDITIONAL_CONFIRM = {
@@ -212,6 +215,26 @@ def confirmation_error(name: str, arguments: dict[str, Any]) -> ToolInputError |
f"such as rm, dd, truncate, or drop): {command!r}. Show the user the exact command, get "
"explicit confirmation, then call again with confirm=true."
)
if name == "db_insert_row":
table = str(arguments.get("table", "")).strip() or "(unspecified)"
return ToolInputError(
f"This writes a new row directly into the '{table}' table. Show the user the exact "
"table and values, get explicit confirmation, then call again with confirm=true."
)
if name == "db_update_row":
table = str(arguments.get("table", "")).strip() or "(unspecified)"
return ToolInputError(
f"This updates an existing row in the '{table}' table directly. Show the user the "
"exact row and new values, get explicit confirmation, then call again with confirm=true."
)
if name == "db_delete_row":
table = str(arguments.get("table", "")).strip() or "(unspecified)"
hard = str(arguments.get("hard", "")).strip().lower() in ("true", "1", "yes", "on")
kind = "PERMANENTLY purges" if hard else "soft-deletes"
return ToolInputError(
f"This {kind} a row in the '{table}' table. Show the user the exact row, get explicit "
"confirmation, then call again with confirm=true."
)
if name in CONFIRM_REQUIRED:
return ToolInputError(
"This removes the item as a soft delete: it disappears from every surface and is only "
@@ -547,7 +570,7 @@ class Dispatcher:
key = self._file_key(arguments)
if key is not None:
self._read_files.add(key)
if action.method in MUTATING_METHODS:
if action.method in MUTATING_METHODS and not action.is_read_only:
record_mutation(action.name)
store = get_store()
if store is not None:
devplacepy/services/devii/agentic/loop.py
+4 -1
View File
@@ -31,7 +31,10 @@ PLAN_VIOLATION = (
VERIFICATION_GATE = (
"[verification-gate] You produced a final answer after performing changes without calling "
"verify(). Confirm the change took effect and call verify() with a summary. If verification "
"truly does not apply, reply explicitly starting with: 'No verification applicable: <reason>'."
"truly does not apply (for example the action only read data and changed nothing), do NOT "
"reply with a bare disclaimer: give the user the full answer they asked for - including any "
"rows or data you retrieved - and you may note 'No verification applicable: <reason>' at the "
"end. Never drop the requested data."
)
ITERATION_LIMIT_MESSAGE = "[stopped] Maximum iterations reached without a final answer."
devplacepy/services/pubsub/__init__.py
+12
View File
@@ -0,0 +1,12 @@
# retoor <retoor@molodetz.nl>
from .hub import pubsub
from .service import PubSubService
__all__ = ["pubsub", "PubSubService", "publish"]
async def publish(topic: str, data) -> int:
return await pubsub.publish(
topic, {"type": "message", "topic": topic, "data": data}
)
devplacepy/services/pubsub/hub.py
+63
View File
@@ -0,0 +1,63 @@
# retoor <retoor@molodetz.nl>
from __future__ import annotations
import logging
logger = logging.getLogger(__name__)
def topic_matches(pattern: str, topic: str) -> bool:
if pattern == topic or pattern == "*":
return True
if pattern.endswith(".*"):
prefix = pattern[:-1]
return topic == pattern[:-2] or topic.startswith(prefix)
return False
class PubSubHub:
def __init__(self) -> None:
self._subscriptions: dict[str, set] = {}
def subscribe(self, pattern: str, socket) -> None:
self._subscriptions.setdefault(pattern, set()).add(socket)
def unsubscribe(self, pattern: str, socket) -> None:
sockets = self._subscriptions.get(pattern)
if not sockets:
return
sockets.discard(socket)
if not sockets:
self._subscriptions.pop(pattern, None)
def drop_socket(self, socket) -> None:
for pattern in list(self._subscriptions):
self.unsubscribe(pattern, socket)
def _targets(self, topic: str) -> set:
targets: set = set()
for pattern, sockets in self._subscriptions.items():
if topic_matches(pattern, topic):
targets.update(sockets)
return targets
async def publish(self, topic: str, frame: dict) -> int:
targets = self._targets(topic)
delivered = 0
for socket in targets:
try:
await socket.send_json(frame)
delivered += 1
except Exception:
logger.debug("pubsub dropping dead socket for %s", topic)
return delivered
def topics(self) -> list[dict]:
return [
{"topic": pattern, "subscribers": len(sockets)}
for pattern, sockets in sorted(self._subscriptions.items())
]
pubsub = PubSubHub()
devplacepy/services/pubsub/policy.py
+76
View File
@@ -0,0 +1,76 @@
# retoor <retoor@molodetz.nl>
from __future__ import annotations
import re
from dataclasses import dataclass
from devplacepy.database import get_setting
from devplacepy.services.dbapi.policy import caller_for as _dbapi_caller
from devplacepy.utils import get_current_user, is_admin
TOPIC_NAME = re.compile(r"^[A-Za-z0-9_.*-]{1,128}$")
MAX_PAYLOAD_BYTES = 64 * 1024
@dataclass(frozen=True)
class Actor:
kind: str
uid: str
username: str
@property
def privileged(self) -> bool:
return self.kind in ("admin", "internal")
def guests_enabled() -> bool:
return get_setting("pubsub_allow_guests", "0") == "1"
def resolve_actor(scope) -> Actor:
caller = _dbapi_caller(scope)
if caller is not None:
return Actor(kind=caller.kind, uid=caller.uid, username=caller.username)
user = get_current_user(scope)
if user:
kind = "admin" if is_admin(user) else "user"
return Actor(kind=kind, uid=user["uid"], username=user.get("username", ""))
return Actor(kind="guest", uid="", username="guest")
def valid_topic(topic: str) -> bool:
return bool(topic and TOPIC_NAME.match(topic))
def _own_namespace(actor: Actor, topic: str) -> bool:
if not actor.uid:
return False
base = f"user.{actor.uid}"
return topic == base or topic.startswith(base + ".")
def _is_public(topic: str) -> bool:
return topic == "public" or topic.startswith("public.") or topic.startswith("public")
def can_subscribe(actor: Actor, topic: str) -> bool:
if actor.privileged:
return True
if topic == "*":
return False
if topic == "public" or topic.startswith("public."):
return actor.kind != "guest" or guests_enabled()
if actor.kind == "user" and _own_namespace(actor, topic):
return True
return False
def can_publish(actor: Actor, topic: str) -> bool:
if actor.privileged:
return True
if actor.kind == "guest":
return False
if actor.kind == "user" and _own_namespace(actor, topic):
return True
return False
devplacepy/services/pubsub/service.py
+36
View File
@@ -0,0 +1,36 @@
# retoor <retoor@molodetz.nl>
from __future__ import annotations
from devplacepy.services.base import BaseService, ConfigField
from .hub import pubsub
class PubSubService(BaseService):
title = "Pub/Sub"
description = (
"In-process publish/subscribe bus over websockets. Lets the frontend, services and "
"Devii broadcast and receive messages on named topics. Database-free and ephemeral; "
"served on the service lock owner so every subscriber converges on one worker."
)
default_enabled = True
def __init__(self):
super().__init__(name="pubsub", interval_seconds=3600)
self.config_fields = [
ConfigField(
"pubsub_allow_guests",
"Allow guests",
type="bool",
default=False,
help="Allow unauthenticated guests to subscribe to public.* topics.",
group="Pub/Sub",
),
]
async def run_once(self) -> None:
return None
def collect_metrics(self) -> dict:
return {"topics": len(pubsub.topics())}
devplacepy/static/css/base.css
+1
View File
@@ -904,6 +904,7 @@ body:has(.page-messages) {
padding-bottom: 0;
flex: 1 1 auto;
min-height: 0;
width: 100%;
display: flex;
}
devplacepy/static/js/Application.js
+2
View File
@@ -30,6 +30,7 @@ import { IssueReporter } from "./IssueReporter.js";
import { MediaGallery } from "./MediaGallery.js";
import WindowManager from "./components/WindowManager.js";
import { ContainerTerminalManager } from "./ContainerTerminalManager.js";
import { PubSubClient } from "./PubSubClient.js";
class Application {
constructor() {
@@ -67,6 +68,7 @@ class Application {
this.projectForker = new ProjectForker();
this.issueReporter = new IssueReporter();
this.mediaGallery = new MediaGallery();
this.pubsub = new PubSubClient();
}
}
devplacepy/static/js/PubSubClient.js
+100
View File
@@ -0,0 +1,100 @@
export class PubSubClient {
constructor() {
this.socket = null;
this.ready = false;
this.handlers = new Map();
this.pending = [];
this.backoff = 200;
this.maxBackoff = 5000;
}
_url() {
const protocol = location.protocol === "https:" ? "wss:" : "ws:";
return `${protocol}//${location.host}/pubsub/ws`;
}
_connect() {
if (this.socket) return;
const socket = new WebSocket(this._url());
this.socket = socket;
socket.addEventListener("open", () => {
this.ready = true;
this.backoff = 200;
for (const topic of this.handlers.keys()) {
socket.send(JSON.stringify({ type: "subscribe", topic }));
}
const queued = this.pending;
this.pending = [];
queued.forEach((frame) => socket.send(JSON.stringify(frame)));
});
socket.addEventListener("message", (event) => {
let frame;
try {
frame = JSON.parse(event.data);
} catch (error) {
return;
}
if (frame.type === "message") this._dispatch(frame);
});
socket.addEventListener("close", (event) => {
this.ready = false;
this.socket = null;
const delay = event.code === 4013 ? 200 : this.backoff;
this.backoff = Math.min(this.backoff * 2, this.maxBackoff);
if (this.handlers.size || this.pending.length) {
setTimeout(() => this._connect(), delay);
}
});
}
_matches(pattern, topic) {
if (pattern === topic || pattern === "*") return true;
if (pattern.endsWith(".*")) {
return topic === pattern.slice(0, -2) || topic.startsWith(pattern.slice(0, -1));
}
return false;
}
_dispatch(frame) {
for (const [pattern, callbacks] of this.handlers.entries()) {
if (this._matches(pattern, frame.topic)) {
callbacks.forEach((callback) => callback(frame.data, frame.topic));
}
}
}
_send(frame) {
if (this.ready && this.socket) {
this.socket.send(JSON.stringify(frame));
} else {
this.pending.push(frame);
this._connect();
}
}
subscribe(topic, callback) {
let callbacks = this.handlers.get(topic);
if (!callbacks) {
callbacks = new Set();
this.handlers.set(topic, callbacks);
this._send({ type: "subscribe", topic });
}
callbacks.add(callback);
this._connect();
return () => this.unsubscribe(topic, callback);
}
unsubscribe(topic, callback) {
const callbacks = this.handlers.get(topic);
if (!callbacks) return;
callbacks.delete(callback);
if (!callbacks.size) {
this.handlers.delete(topic);
this._send({ type: "unsubscribe", topic });
}
}
publish(topic, data) {
this._send({ type: "publish", topic, data });
}
}
devplacepy/templates/docs/services-dbapi.html
+133
View File
@@ -0,0 +1,133 @@
<div class="docs-content" data-render>
# Database API service
The database API is a single, safe surface for reading and updating **every table** in the platform. It exposes generic CRUD per table, a validated read-only `query()`, a natural-language-to-SQL designer backed by the platform AI gateway, and asynchronous query execution streamed over a websocket. It is mounted at `/dbapi` and is reachable only by administrators or by internal services.
> Audience: administrators, maintainers, and trusted internal services.
It reuses the existing data layer (`dataset` with the production pragmas), the [async job framework](/docs/architecture-jobs.html), the [AI gateway](/docs/services-gateway.html), and the Devii action catalog. It does not introduce a second database or a new ORM.
## Who can call it
There is exactly one authorization boundary, enforced on every route:
- An **administrator** authenticated by session cookie or by an admin API key.
- An **internal service** presenting the gateway internal key as `Authorization: Bearer <key>` or `X-API-KEY: <key>`.
Anyone else (members, guests) receives `403 Forbidden`, and the denial is written to the audit log as `database.access.denied`. There is no public or member-facing access.
## Tables and the deny list
Every table-scoped route runs through a guard that validates the table name against a strict pattern, confirms the table exists, and rejects any table on the deny list. The deny list always contains the credential and session tables (`sessions`, `password_resets`, `cache_state`) and can be extended by the `dbapi_deny_tables` setting on the service config. The guard protects both the path segment and the table names referenced by a designed or submitted SQL query, so neither a crafted URL nor an AI-designed query can reach a denied table.
```
GET /dbapi/tables
{ "tables": [ { "name": "posts", "row_count": 1240, "soft_delete": true }, ... ], "count": 49 }
GET /dbapi/posts/schema
{ "table": "posts", "columns": [ { "name": "uid", "type": "TEXT" }, ... ], "soft_delete": true, "row_count": 1240 }
```
## CRUD per table
All writes go through the structured CRUD, which is parameterized, soft-delete aware, and never executes raw SQL. The endpoints are:
- `GET /dbapi/{table}` lists rows newest-first with keyset pagination. Filter with `?filter.<col>=value` (equality) or `?gte.<col>=`, `?lte.<col>=`, `?gt.<col>=`, `?lt.<col>=` (comparisons), full-text search common columns with `?search=`, page with `?before=<cursor>` and `?limit=` (max 500), and include soft-deleted rows with `?include_deleted=true`.
- `GET /dbapi/{table}/{key}/{value}` returns one row where the key column equals the value (the key is usually `uid`).
- `POST /dbapi/{table}` inserts a row. The body is the column map. Inserts are **born live**: `uid` and `created_at` are filled automatically when those columns exist, and soft-delete tables get `deleted_at: null` and `deleted_by: null` so the row is visible immediately. **Unknown columns are rejected**, so the API can never grow or pollute the schema.
- `PATCH /dbapi/{table}/{key}/{value}` updates a row. `uid` and `id` can never be changed, and `updated_at` is set when present.
- `DELETE /dbapi/{table}/{key}/{value}` removes a row. By default this is a **soft delete** (the row is stamped and disappears from normal reads but stays restorable from Trash). Pass `?hard=true` to permanently purge it, or for tables that have no soft-delete columns.
- `POST /dbapi/{table}/{key}/{value}/restore` clears a soft delete.
Every mutation writes an audit event (`database.row.insert`, `.update`, `.delete`, `.restore`).
```
POST /dbapi/bookmarks { "user_uid": "u1", "target_type": "post", "target_uid": "p1" }
-> { "table": "bookmarks", "ok": true, "mode": "insert", "row": { "uid": "...", "deleted_at": null, ... } }
DELETE /dbapi/bookmarks/uid/<uid> -> { "ok": true, "mode": "soft", ... }
POST /dbapi/bookmarks/uid/<uid>/restore -> { "ok": true, "mode": "restore", ... }
DELETE /dbapi/bookmarks/uid/<uid>?hard=true -> { "ok": true, "mode": "hard", ... }
```
## Read-only query()
`POST /dbapi/query` runs a single SQL `SELECT` and returns the rows. It is **hard SELECT-only**: any other statement is refused. Validation runs in three stages before a query executes:
1. **Parse and classify** with `sqlglot`: determine the statement type, the tables referenced, and whether the query has a `WHERE`, a `JOIN`, and a `LIMIT`.
2. **Flag suspicious shapes**: a `SELECT` with no `WHERE`, `JOIN`, or `LIMIT` (which scans an entire table), multiple statements, or `ATTACH`/`PRAGMA`/`VACUUM` style statements.
3. **Dry run** the statement with `EXPLAIN` on a **separate read-only connection** (opened `mode=ro` with `PRAGMA query_only=ON`), which validates the SQL against the real schema without executing its body.
A non-`SELECT` returns `409 Conflict` with a hint to use the CRUD routes; an invalid `SELECT` returns `400`; a valid query returns the rows plus a `suspicious` list. Execution itself also happens on the read-only connection and is capped at `dbapi_max_rows`.
```
POST /dbapi/query { "sql": "SELECT uid, username FROM users WHERE role = 'Admin' LIMIT 20" }
-> { "sql": "...", "valid": true, "rows": [ ... ], "row_count": 3, "truncated": false, "suspicious": [] }
POST /dbapi/query { "sql": "DELETE FROM posts" }
-> 409 { "valid": false, "statement_type": "delete", "error": "Only SELECT queries run through query(); ..." }
POST /dbapi/query { "sql": "SELECT * FROM users" }
-> 200 { "valid": true, "suspicious": ["SELECT has no WHERE, JOIN, or LIMIT and may return an entire table."], ... }
```
Mutations are never possible through `query()`. To change data, use the structured CRUD routes above.
## Ask in plain language
`POST /dbapi/nl` turns a natural-language question about one table into a validated `SELECT`. The designer builds a system prompt from the table schema and a handful of example rows, asks the AI gateway to write the query, and then **re-prompts the model with the validator's error until the SQL validates** (up to three attempts). For soft-delete tables it instructs the model to add `deleted_at IS NULL` unless `apply_soft_delete` is set to false.
By default it returns only the SQL; pass `execute: true` to also run it read-only and include the rows.
```
POST /dbapi/nl
{ "question": "all users registered longer than three days", "table": "users", "execute": true }
->
{
"sql": "SELECT * FROM users WHERE created_at < '...' AND deleted_at IS NULL",
"valid": true, "attempts": 1, "applied_soft_delete": true,
"executed": true, "rows": [ ... ], "row_count": 12
}
```
The model is configurable (`dbapi_nl_model`, blank uses the internal `molodetz` model), as is an optional operator preamble (`dbapi_nl_system_preamble`). The call is attributed to the calling administrator's API key so its cost rolls up under that user.
## Asynchronous queries
For heavy or large result sets, run the query off the request path:
- `POST /dbapi/query/async` validates the SQL, enqueues a `dbquery` job, and returns `{ uid, status_url, ws_url }`.
- `GET /dbapi/query/{uid}` returns the job status.
- `GET /dbapi/query/{uid}/result` returns the full result set (read from disk) and extends the retention window.
- `WS /dbapi/query/{uid}/ws` streams live progress. Like every job websocket it is served only by the service lock owner: a non-owner worker closes with code `4013` and the client retries until it lands on the owner. On connect the socket replays any buffered frames, then streams `progress` frames and a terminal `done` (or `failed`) frame.
The job writes its result to the runtime data directory (`config.DBAPI_DIR/{uid}/result.json`), outside the package, and the result is removed when the job's retention expires.
## Devii
An administrator's Devii assistant exposes the same capability conversationally through admin-only tools:
- Read: `db_list_tables`, `db_table_schema`, `db_list_rows`, `db_get_row`, `db_query` (SELECT only, and it surfaces any `suspicious` warnings), and `db_design_query` (natural language to SQL).
- Write: `db_insert_row`, `db_update_row`, `db_delete_row`. Each is **confirmation gated**: the first call is refused and Devii must show the user exactly what will change and obtain explicit confirmation before calling again with `confirm=true`. The mutation tools pass arbitrary columns as a JSON object string in `values_json`.
This is how the rule "a non-SELECT is always confirmed" is honored: raw SQL stays read-only, and every write is a confirm-gated CRUD tool.
## Configuration
On `/admin/services` the **Database API** service (`dbquery`) exposes:
- `dbapi_max_rows` - hard cap on rows returned by any query (default 5000).
- `dbapi_nl_model` - model used to design SQL from natural language (blank uses `molodetz`).
- `dbapi_nl_system_preamble` - optional operator text prepended to the NL-to-SQL prompt.
- `dbapi_deny_tables` - comma separated extra tables to hide.
- The standard job fields: artifact retention, maximum concurrent jobs, and job timeout.
## Security summary
- One authorization boundary: administrator or internal key, enforced on every route, audited on denial.
- Table allow/deny guard on every table-scoped path and on every table referenced by a query.
- Raw SQL is always read-only, on a dedicated `query_only` connection, so even a validator miss cannot mutate.
- CRUD rejects unknown columns, so the API never alters the schema.
- Inserts are born live, deletes are soft and audited, reads exclude soft-deleted rows by default.
- Devii read tools are admin-only; Devii write tools are admin-only and confirmation gated.
</div>
devplacepy/templates/docs/services-pubsub.html
+89
View File
@@ -0,0 +1,89 @@
<div class="docs-content" data-render>
# Pub/Sub service
The pub/sub service is a general-purpose, **database-free** publish/subscribe bus over websockets. It lets the frontend, background services, and Devii broadcast and receive messages on named topics without polling. It is mounted at `/pubsub`.
> Audience: administrators, maintainers, and frontend developers.
It reuses the in-process hub pattern used elsewhere in the platform (the same shape as the job progress hub) and the service lock-owner gating used by Devii and the tools websockets. It stores nothing: there is no table, no message log, and no persistence.
## How delivery works without a database
Messages are delivered entirely in memory. The hub keeps a map from topic pattern to the set of live websockets subscribed to it; publishing a message sends it to every matching socket and drops any socket that has gone away.
To make this correct across multiple web workers without a message broker or a database, the bus is **served on the service lock owner only**. The `WS /pubsub/ws` handler closes with code `4013` on any non-owner worker, and the browser client transparently reconnects until it lands on the owner. Because every subscriber converges on the one owner worker, an in-process fan-out reaches all of them. Background services, which already run on the lock owner, publish in-process.
This is a deliberate trade-off. The bus is **ephemeral and best-effort**: a publish reaches whoever is connected at that moment, and a server restart drops in-flight messages. It is built for live UI updates and coordination, not for durable messaging. If you need durability, persist the event yourself (for example with the [database API](/docs/services-dbapi.html)) and publish a notification on the bus.
## Topics and wildcards
A topic is a dotted name such as `public.deploys` or `user.<uid>.inbox`, matching `^[A-Za-z0-9_.*-]{1,128}$`. A subscription may use a trailing wildcard: subscribing to `foo.*` receives `foo` and any `foo.bar`. The client dispatches wildcard matches locally as well, so a single subscription callback fires for every matching topic.
## The websocket protocol
Connect to `WS /pubsub/ws`. The server first sends `{ "type": "ready", "actor": "<kind>" }`. Then send frames:
```
{ "type": "subscribe", "topic": "public.deploys" }
{ "type": "unsubscribe", "topic": "public.deploys" }
{ "type": "publish", "topic": "public.deploys", "data": { "service": "web", "status": "ok" } }
```
The server replies with `{"type":"subscribed"}` / `{"type":"unsubscribed"}` / `{"type":"ack","delivered":N}`, or `{"type":"error","message":"..."}` when a topic is invalid or not allowed. A delivered message arrives as:
```
{ "type": "message", "topic": "public.deploys", "data": { ... }, "ts": "2026-..." }
```
Payloads are capped at 64 KB and are never evaluated; they are forwarded verbatim.
## Who can do what
The connecting identity is resolved the same way as the database API (admin or internal key) and then falls back to the session user or a guest. Authorization is checked on every frame:
- **Administrators and internal services** may subscribe and publish to any topic.
- **Members** may subscribe to `public.*` and to their own `user.<uid>.*` namespace, and may publish only to their own namespace. Publishing to `public.*` is restricted to administrators and internal services so a member cannot broadcast to everyone.
- **Guests** get read-only access to `public.*` topics, and only when `pubsub_allow_guests` is enabled on the service config; otherwise the socket is closed.
## Publishing from the backend and over HTTP
Background code on the lock owner publishes in-process:
```python
from devplacepy.services import pubsub
await pubsub.publish("public.deploys", {"service": "web", "status": "ok"})
```
Administrators and internal services can also publish over plain HTTP, which is useful from scripts and from Devii:
- `POST /pubsub/publish` with `{ "topic": "...", "data": ... }` publishes a message and returns the delivered count. Because the bus lives on the lock owner, a request that lands on a non-owner worker returns `409` and should be retried.
- `GET /pubsub/topics` lists the live topics and their subscriber counts.
Both are admin/internal only, and a publish is recorded in the audit log as `pubsub.publish`.
## Frontend client
The browser client is created once as `app.pubsub` (`static/js/PubSubClient.js`). It connects lazily on first use, reconnects with backoff (and a fast retry on the `4013` owner bounce), and re-subscribes to all topics after a reconnect.
```js
const off = app.pubsub.subscribe("public.deploys", (data, topic) => {
app.toast.show(`${data.service}: ${data.status}`);
});
app.pubsub.publish("user." + myUid + ".ping", { at: Date.now() });
off(); // unsubscribe
```
## Configuration
On `/admin/services` the **Pub/Sub** service exposes a single toggle, `pubsub_allow_guests`, which lets unauthenticated guests subscribe to `public.*` topics. Disabling the service closes the websocket with the standard "try again later" code. The service has no run loop; it exists to surface the toggle and to gate the websocket.
## Security summary
- The websocket requires a resolvable identity; guests are admitted only when explicitly enabled.
- Every subscribe and publish is authorized per frame against the topic namespace rules.
- HTTP publish and topic introspection are admin/internal only.
- Topic names are validated and payloads are size-capped; nothing is persisted, so there is no data at rest.
</div>
devplacepy/templates/gist_detail.html
+2
View File
@@ -51,6 +51,8 @@
{% set _type = "gist" %}{% set _uid = gist['uid'] %}{% set _bookmarked = bookmarked %}{% include "_bookmark_button.html" %}
{% if is_owner %}
<button type="button" class="gist-star-btn" data-modal="edit-gist-modal"><span class="icon">&#x270F;&#xFE0F;</span>Edit</button>
{% endif %}
{% if is_owner or is_admin(user) %}
<form method="POST" action="/gists/delete/{{ gist['slug'] or gist['uid'] }}" style="display:inline;">
<button type="submit" class="gist-star-btn" data-confirm="Delete this gist?"><span class="icon">&#x1F5D1;&#xFE0F;</span>Delete</button>
</form>
devplacepy/templates/post.html
+2
View File
@@ -45,6 +45,8 @@
{% set _type = "post" %}{% set _uid = post['uid'] %}{% set _bookmarked = bookmarked %}{% include "_bookmark_button.html" %}
{% if is_owner %}
<button type="button" class="post-action-btn" data-modal="edit-post-modal"><span class="icon">&#x270F;&#xFE0F;</span>Edit</button>
{% endif %}
{% if is_owner or is_admin(user) %}
<form method="POST" action="/posts/delete/{{ post['slug'] or post['uid'] }}" class="inline-form">
<button type="submit" class="post-action-btn" data-confirm="Delete this post?"><span class="icon">&#x1F5D1;&#xFE0F;</span>Delete</button>
</form>
devplacepy/templates/project_detail.html
+2
View File
@@ -94,6 +94,8 @@
<input type="hidden" name="value" value="{{ 0 if read_only else 1 }}">
<button type="submit" data-confirm-danger data-confirm="{% if read_only %}Allow file changes again for this project?{% else %}Make this project read-only? Files become immutable until you turn this off.{% endif %}" data-menu-action data-menu-icon="{% if read_only %}&#x1F4DD;{% else %}&#x1F6AB;{% endif %}" data-menu-label="{% if read_only %}Allow edits{% else %}Make read-only{% endif %}">{% if read_only %}Allow edits{% else %}Make read-only{% endif %}</button>
</form>
{% endif %}
{% if is_owner or is_admin(user) %}
<form method="POST" action="/projects/delete/{{ project['slug'] or project['uid'] }}">
<button type="submit" data-confirm-danger data-confirm="Delete this project?" data-menu-action data-menu-icon="&#x1F5D1;&#xFE0F;" data-menu-label="Delete">Delete</button>
</form>
devplacepy/templating.py
+2 -1
View File
@@ -13,7 +13,8 @@ from devplacepy.attachments import format_file_size, file_icon_emoji
from devplacepy.content import is_owner as _owns
from devplacepy.customization import custom_css_tag, custom_js_tag, page_type_for
templates = Jinja2Templates(directory=str(TEMPLATES_DIR), auto_reload=TEMPLATE_AUTO_RELOAD)
templates = Jinja2Templates(directory=str(TEMPLATES_DIR))
templates.env.auto_reload = TEMPLATE_AUTO_RELOAD
def is_self(user, uid) -> bool:
devplacepy/utils.py
+1 -1
View File
@@ -325,7 +325,7 @@ def generate_uid() -> str:
def make_combined_slug(text: str, uid: str) -> str:
short_uid = uid[:8]
short_uid = uid.replace("-", "")[-12:]
slug_part = slugify(text)
if not slug_part:
return short_uid