feat: add admin/internal database API with CRUD, read-only query, and natural-language SQL endpoints
Add a new `/dbapi` router package providing a generic database API over `dataset`, restricted to admin sessions, admin API keys, and the internal gateway key. Includes: - `tables.py`: list all tables and inspect table schemas - `crud.py`: full CRUD operations (GET, POST, PATCH, DELETE) with soft-delete awareness, born-live inserts, `?include_deleted`, `.../restore`, and `?hard=true` purge - `query.py`: validated read-only SELECT execution via sqlglot parsing, classification, and EXPLAIN dry-run; async query jobs with WebSocket streaming via `DbApiJobService` - `nl.py`: natural-language-to-SQL conversion using the platform AI gateway with re-prompting until validation passes Also register `DbApiJobService` and `PubSubService` in the service manager, add `DBAPI_DIR` to config data paths, and force cleartext `http://` connections to HTTP/1.1 in `curl_transport` to fix large request failures against uvicorn's HTTP/1.1-only internal gateway.
This commit is contained in:
@@ -0,0 +1,11 @@
|
||||
# retoor <retoor@molodetz.nl>
|
||||
|
||||
from fastapi import APIRouter
|
||||
|
||||
from . import crud, nl, query, tables
|
||||
|
||||
router = APIRouter()
|
||||
router.include_router(tables.router)
|
||||
router.include_router(query.router)
|
||||
router.include_router(nl.router)
|
||||
router.include_router(crud.router)
|
||||
@@ -0,0 +1,79 @@
|
||||
# retoor <retoor@molodetz.nl>
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from fastapi import HTTPException, Request
|
||||
from fastapi.responses import JSONResponse
|
||||
|
||||
from devplacepy.services.dbapi import policy
|
||||
from devplacepy.services.dbapi.policy import Caller, DbApiBadTable, DbApiDenied
|
||||
|
||||
OPERATORS = {"gte": ">=", "lte": "<=", "gt": ">", "lt": "<"}
|
||||
RESERVED = {"limit", "before", "search", "include_deleted", "key", "hard", "execute"}
|
||||
|
||||
|
||||
def require_dbapi_caller(request: Request) -> Caller:
|
||||
try:
|
||||
return policy.require_caller(request)
|
||||
except DbApiDenied as exc:
|
||||
from devplacepy.services.audit import record as audit
|
||||
|
||||
audit.record(
|
||||
request,
|
||||
"database.access.denied",
|
||||
result="denied",
|
||||
summary=f"denied database API access to {request.url.path}",
|
||||
)
|
||||
raise HTTPException(status_code=403, detail=str(exc))
|
||||
|
||||
|
||||
def assert_table(name: str):
|
||||
try:
|
||||
return policy.assert_table(name)
|
||||
except DbApiBadTable as exc:
|
||||
raise HTTPException(status_code=404, detail=str(exc))
|
||||
|
||||
|
||||
def error(status: int, message: str, **extra) -> JSONResponse:
|
||||
return JSONResponse(
|
||||
{"error": {"status": status, "message": message, **extra}}, status_code=status
|
||||
)
|
||||
|
||||
|
||||
async def read_body(request: Request) -> dict:
|
||||
content_type = request.headers.get("content-type", "")
|
||||
if content_type.startswith("application/json"):
|
||||
try:
|
||||
data = await request.json()
|
||||
except Exception:
|
||||
return {}
|
||||
return data if isinstance(data, dict) else {}
|
||||
form = await request.form()
|
||||
return {key: value for key, value in form.items()}
|
||||
|
||||
|
||||
def row_payload(body: dict) -> dict:
|
||||
if "values_json" in body:
|
||||
import json
|
||||
|
||||
try:
|
||||
parsed = json.loads(body["values_json"])
|
||||
except Exception:
|
||||
return {}
|
||||
return parsed if isinstance(parsed, dict) else {}
|
||||
return {key: value for key, value in body.items() if key not in ("confirm", "values_json")}
|
||||
|
||||
|
||||
def parse_filters(request: Request) -> tuple[dict, dict]:
|
||||
filters: dict = {}
|
||||
comparisons: dict = {}
|
||||
for key, value in request.query_params.multi_items():
|
||||
if key in RESERVED:
|
||||
continue
|
||||
if "." in key:
|
||||
prefix, column = key.split(".", 1)
|
||||
if prefix in ("filter", "eq"):
|
||||
filters[column] = value
|
||||
elif prefix in OPERATORS:
|
||||
comparisons[column] = {OPERATORS[prefix]: value}
|
||||
return filters, comparisons
|
||||
@@ -0,0 +1,152 @@
|
||||
# retoor <retoor@molodetz.nl>
|
||||
|
||||
from fastapi import APIRouter, Request
|
||||
from fastapi.responses import JSONResponse
|
||||
|
||||
from devplacepy.schemas import DbMutationOut, DbRowOut, DbRowsOut
|
||||
from devplacepy.services.dbapi import crud
|
||||
from devplacepy.services.dbapi.crud import DbApiError
|
||||
|
||||
from ._shared import (
|
||||
assert_table,
|
||||
error,
|
||||
parse_filters,
|
||||
read_body,
|
||||
require_dbapi_caller,
|
||||
row_payload,
|
||||
)
|
||||
|
||||
router = APIRouter()
|
||||
|
||||
|
||||
def _target_uid(row: dict, value: str) -> str:
|
||||
return (row or {}).get("uid") or str(value)
|
||||
|
||||
|
||||
@router.get("/{table}")
|
||||
async def dbapi_list(
|
||||
request: Request,
|
||||
table: str,
|
||||
limit: int = 25,
|
||||
before: str = None,
|
||||
search: str = "",
|
||||
include_deleted: bool = False,
|
||||
):
|
||||
require_dbapi_caller(request)
|
||||
assert_table(table)
|
||||
filters, comparisons = parse_filters(request)
|
||||
rows, next_cursor = crud.list_rows(
|
||||
table,
|
||||
filters=filters,
|
||||
comparisons=comparisons,
|
||||
search=search,
|
||||
before=before,
|
||||
limit=limit,
|
||||
include_deleted=include_deleted,
|
||||
)
|
||||
return JSONResponse(
|
||||
DbRowsOut(
|
||||
table=table, rows=rows, count=len(rows), next_cursor=next_cursor
|
||||
).model_dump(mode="json")
|
||||
)
|
||||
|
||||
|
||||
@router.get("/{table}/{key}/{value}")
|
||||
async def dbapi_get(request: Request, table: str, key: str, value: str):
|
||||
require_dbapi_caller(request)
|
||||
assert_table(table)
|
||||
try:
|
||||
row = crud.get_row(table, key, value)
|
||||
except DbApiError as exc:
|
||||
return error(400, str(exc))
|
||||
if row is None:
|
||||
return error(404, "Row not found")
|
||||
return JSONResponse(DbRowOut(table=table, row=row).model_dump(mode="json"))
|
||||
|
||||
|
||||
@router.post("/{table}")
|
||||
async def dbapi_insert(request: Request, table: str):
|
||||
caller = require_dbapi_caller(request)
|
||||
assert_table(table)
|
||||
body = await read_body(request)
|
||||
try:
|
||||
row = crud.insert_row(table, row_payload(body), caller.uid)
|
||||
except DbApiError as exc:
|
||||
return error(400, str(exc))
|
||||
_audit(request, "database.row.insert", table, _target_uid(row, ""), caller)
|
||||
return JSONResponse(
|
||||
DbMutationOut(table=table, ok=True, mode="insert", row=row).model_dump(mode="json")
|
||||
)
|
||||
|
||||
|
||||
@router.patch("/{table}/{key}/{value}")
|
||||
async def dbapi_update(request: Request, table: str, key: str, value: str):
|
||||
caller = require_dbapi_caller(request)
|
||||
assert_table(table)
|
||||
body = await read_body(request)
|
||||
try:
|
||||
row = crud.update_row(table, key, value, row_payload(body), caller.uid)
|
||||
except DbApiError as exc:
|
||||
return error(400, str(exc))
|
||||
if row is None:
|
||||
return error(404, "Row not found")
|
||||
_audit(request, "database.row.update", table, _target_uid(row, value), caller)
|
||||
return JSONResponse(
|
||||
DbMutationOut(table=table, ok=True, mode="update", row=row).model_dump(mode="json")
|
||||
)
|
||||
|
||||
|
||||
@router.delete("/{table}/{key}/{value}")
|
||||
async def dbapi_delete(
|
||||
request: Request, table: str, key: str, value: str, hard: bool = False
|
||||
):
|
||||
caller = require_dbapi_caller(request)
|
||||
assert_table(table)
|
||||
try:
|
||||
outcome = crud.delete_row(table, key, value, caller.uid, hard=hard)
|
||||
except DbApiError as exc:
|
||||
return error(400, str(exc))
|
||||
if outcome is None:
|
||||
return error(404, "Row not found")
|
||||
_audit(
|
||||
request,
|
||||
"database.row.delete",
|
||||
table,
|
||||
_target_uid(outcome.get("row", {}), value),
|
||||
caller,
|
||||
metadata={"mode": outcome["mode"]},
|
||||
)
|
||||
return JSONResponse(
|
||||
DbMutationOut(
|
||||
table=table, ok=True, mode=outcome["mode"], row=outcome.get("row")
|
||||
).model_dump(mode="json")
|
||||
)
|
||||
|
||||
|
||||
@router.post("/{table}/{key}/{value}/restore")
|
||||
async def dbapi_restore(request: Request, table: str, key: str, value: str):
|
||||
caller = require_dbapi_caller(request)
|
||||
assert_table(table)
|
||||
try:
|
||||
row = crud.restore_row(table, key, value, caller.uid)
|
||||
except DbApiError as exc:
|
||||
return error(400, str(exc))
|
||||
if row is None:
|
||||
return error(404, "Row not found")
|
||||
_audit(request, "database.row.restore", table, _target_uid(row, value), caller)
|
||||
return JSONResponse(
|
||||
DbMutationOut(table=table, ok=True, mode="restore", row=row).model_dump(mode="json")
|
||||
)
|
||||
|
||||
|
||||
def _audit(request, event_key, table, target_uid, caller, metadata=None):
|
||||
from devplacepy.services.audit import record as audit
|
||||
|
||||
audit.record(
|
||||
request,
|
||||
event_key,
|
||||
target_type=table,
|
||||
target_uid=target_uid,
|
||||
summary=f"{caller.username or caller.kind} {event_key} on {table}/{target_uid}",
|
||||
metadata={"table": table, "caller": caller.kind, **(metadata or {})},
|
||||
)
|
||||
@@ -0,0 +1,65 @@
|
||||
# retoor <retoor@molodetz.nl>
|
||||
|
||||
from fastapi import APIRouter, Request
|
||||
from fastapi.responses import JSONResponse
|
||||
|
||||
from devplacepy.database import get_int_setting
|
||||
from devplacepy.schemas import NlQueryOut
|
||||
from devplacepy.services.dbapi import nl2sql
|
||||
from devplacepy.services.dbapi.validate import run_select
|
||||
|
||||
from ._shared import assert_table, error, read_body, require_dbapi_caller
|
||||
|
||||
router = APIRouter()
|
||||
|
||||
DEFAULT_MAX_ROWS = 5000
|
||||
|
||||
|
||||
def _truthy(value) -> bool:
|
||||
return str(value).strip().lower() in ("1", "true", "yes", "on")
|
||||
|
||||
|
||||
@router.post("/nl")
|
||||
async def dbapi_nl(request: Request):
|
||||
caller = require_dbapi_caller(request)
|
||||
body = await read_body(request)
|
||||
question = str(body.get("question", "")).strip()
|
||||
table = str(body.get("table", "")).strip()
|
||||
if not question or not table:
|
||||
return error(400, "Provide both 'question' and 'table'.")
|
||||
assert_table(table)
|
||||
apply_soft_delete = _truthy(body.get("apply_soft_delete", True))
|
||||
dialect = str(body.get("dialect", "sqlite")).strip() or "sqlite"
|
||||
execute = _truthy(body.get("execute", False))
|
||||
|
||||
design = await nl2sql.design_query(
|
||||
question,
|
||||
table,
|
||||
apply_soft_delete=apply_soft_delete,
|
||||
dialect=dialect,
|
||||
api_key=caller.api_key,
|
||||
)
|
||||
out = NlQueryOut(**design.as_dict())
|
||||
|
||||
if execute and design.valid:
|
||||
max_rows = max(1, get_int_setting("dbapi_max_rows", DEFAULT_MAX_ROWS))
|
||||
rows, truncated = run_select(design.sql, max_rows)
|
||||
out.executed = True
|
||||
out.rows = rows
|
||||
out.row_count = len(rows)
|
||||
out.truncated = truncated
|
||||
|
||||
_audit_nl(request, caller, table, design.sql, design.valid)
|
||||
return JSONResponse(out.model_dump(mode="json"))
|
||||
|
||||
|
||||
def _audit_nl(request, caller, table, sql, valid):
|
||||
from devplacepy.services.audit import record as audit
|
||||
|
||||
audit.record(
|
||||
request,
|
||||
"database.nl.design",
|
||||
target_type=table,
|
||||
summary=f"{caller.username or caller.kind} designed a query on {table}",
|
||||
metadata={"table": table, "sql": sql[:500], "valid": valid, "caller": caller.kind},
|
||||
)
|
||||
@@ -0,0 +1,202 @@
|
||||
# retoor <retoor@molodetz.nl>
|
||||
|
||||
import json
|
||||
import logging
|
||||
|
||||
from fastapi import APIRouter, Request, WebSocket, WebSocketDisconnect
|
||||
from fastapi.responses import JSONResponse
|
||||
|
||||
from devplacepy.config import DBAPI_DIR
|
||||
from devplacepy.database import get_int_setting
|
||||
from devplacepy.schemas import DbQueryJobOut, DbQueryOut
|
||||
from devplacepy.services.dbapi import policy
|
||||
from devplacepy.services.dbapi.progress import hub
|
||||
from devplacepy.services.dbapi.validate import run_select, validate_select
|
||||
from devplacepy.services.jobs import queue
|
||||
from devplacepy.services.manager import service_manager
|
||||
|
||||
from ._shared import error, read_body, require_dbapi_caller
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
router = APIRouter()
|
||||
|
||||
TERMINAL_STATES = (queue.DONE, queue.FAILED)
|
||||
DEFAULT_MAX_ROWS = 5000
|
||||
|
||||
|
||||
def _owner(caller) -> tuple[str, str]:
|
||||
if caller.kind == "admin":
|
||||
return "user", caller.uid
|
||||
return "internal", caller.uid
|
||||
|
||||
|
||||
def _max_rows() -> int:
|
||||
return max(1, get_int_setting("dbapi_max_rows", DEFAULT_MAX_ROWS))
|
||||
|
||||
|
||||
@router.post("/query")
|
||||
async def dbapi_query(request: Request):
|
||||
caller = require_dbapi_caller(request)
|
||||
body = await read_body(request)
|
||||
sql = str(body.get("sql", "")).strip()
|
||||
dialect = str(body.get("dialect", "sqlite")).strip() or "sqlite"
|
||||
if not sql:
|
||||
return error(400, "Provide a 'sql' SELECT statement.")
|
||||
verdict = validate_select(sql, dialect=dialect)
|
||||
if not verdict.valid:
|
||||
status = 409 if not verdict.is_select else 400
|
||||
hint = (
|
||||
" Use the structured /dbapi/{table} CRUD routes to change data."
|
||||
if not verdict.is_select
|
||||
else ""
|
||||
)
|
||||
return JSONResponse(
|
||||
DbQueryOut(
|
||||
sql=verdict.sql,
|
||||
valid=False,
|
||||
statement_type=verdict.statement_type,
|
||||
suspicious=verdict.suspicious,
|
||||
error=(verdict.error or "Invalid query.") + hint,
|
||||
).model_dump(mode="json"),
|
||||
status_code=status,
|
||||
)
|
||||
rows, truncated = run_select(verdict.sql, _max_rows())
|
||||
_audit_query(request, caller, verdict.sql, len(rows))
|
||||
return JSONResponse(
|
||||
DbQueryOut(
|
||||
sql=verdict.sql,
|
||||
valid=True,
|
||||
statement_type="select",
|
||||
rows=rows,
|
||||
row_count=len(rows),
|
||||
truncated=truncated,
|
||||
suspicious=verdict.suspicious,
|
||||
).model_dump(mode="json")
|
||||
)
|
||||
|
||||
|
||||
@router.post("/query/async")
|
||||
async def dbapi_query_async(request: Request):
|
||||
caller = require_dbapi_caller(request)
|
||||
body = await read_body(request)
|
||||
sql = str(body.get("sql", "")).strip()
|
||||
if not sql:
|
||||
return error(400, "Provide a 'sql' SELECT statement.")
|
||||
verdict = validate_select(sql)
|
||||
if not verdict.valid:
|
||||
status = 409 if not verdict.is_select else 400
|
||||
return JSONResponse(
|
||||
DbQueryOut(
|
||||
sql=verdict.sql,
|
||||
valid=False,
|
||||
statement_type=verdict.statement_type,
|
||||
suspicious=verdict.suspicious,
|
||||
error=verdict.error or "Invalid query.",
|
||||
).model_dump(mode="json"),
|
||||
status_code=status,
|
||||
)
|
||||
owner_kind, owner_id = _owner(caller)
|
||||
uid = queue.enqueue(
|
||||
"dbquery", {"sql": verdict.sql}, owner_kind, owner_id, "dbquery"
|
||||
)
|
||||
_audit_query(request, caller, verdict.sql, None, event="database.query.async")
|
||||
return JSONResponse(
|
||||
{
|
||||
"uid": uid,
|
||||
"status_url": f"/dbapi/query/{uid}",
|
||||
"ws_url": f"/dbapi/query/{uid}/ws",
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
@router.get("/query/{uid}")
|
||||
async def dbapi_query_status(request: Request, uid: str):
|
||||
require_dbapi_caller(request)
|
||||
job = queue.get_job(uid)
|
||||
if not job or job.get("kind") != "dbquery":
|
||||
return error(404, "Query job not found")
|
||||
result = job.get("result", {})
|
||||
done = job.get("status") == queue.DONE
|
||||
return JSONResponse(
|
||||
DbQueryJobOut(
|
||||
uid=uid,
|
||||
kind="dbquery",
|
||||
status=job.get("status", ""),
|
||||
ws_url=f"/dbapi/query/{uid}/ws",
|
||||
result_url=f"/dbapi/query/{uid}/result" if done else None,
|
||||
row_count=result.get("row_count") if done else None,
|
||||
truncated=result.get("truncated") if done else None,
|
||||
error=job.get("error") or None,
|
||||
created_at=job.get("created_at"),
|
||||
completed_at=job.get("completed_at") or None,
|
||||
).model_dump(mode="json")
|
||||
)
|
||||
|
||||
|
||||
@router.get("/query/{uid}/result")
|
||||
async def dbapi_query_result(request: Request, uid: str):
|
||||
require_dbapi_caller(request)
|
||||
job = queue.get_job(uid)
|
||||
if not job or job.get("kind") != "dbquery":
|
||||
return error(404, "Query job not found")
|
||||
path = (DBAPI_DIR / uid / "result.json").resolve()
|
||||
if not path.is_relative_to(DBAPI_DIR.resolve()) or not path.is_file():
|
||||
return error(404, "Result not available")
|
||||
queue.touch_job(uid, get_int_setting("dbquery_retention_seconds", 604800))
|
||||
return JSONResponse(json.loads(path.read_text(encoding="utf-8")))
|
||||
|
||||
|
||||
@router.websocket("/query/{uid}/ws")
|
||||
async def dbapi_query_ws(websocket: WebSocket, uid: str):
|
||||
await websocket.accept()
|
||||
if not service_manager.owns_lock():
|
||||
await websocket.close(code=4013)
|
||||
return
|
||||
if policy.caller_for(websocket) is None:
|
||||
await websocket.close(code=1008)
|
||||
return
|
||||
job = queue.get_job(uid)
|
||||
if not job or job.get("kind") != "dbquery":
|
||||
await websocket.close(code=1008)
|
||||
return
|
||||
listener = hub.subscribe(uid)
|
||||
try:
|
||||
for frame in hub.snapshot(uid):
|
||||
await websocket.send_json(frame)
|
||||
current = queue.get_job(uid)
|
||||
if current and current.get("status") in TERMINAL_STATES:
|
||||
done = current.get("status") == queue.DONE
|
||||
result = current.get("result", {})
|
||||
await websocket.send_json(
|
||||
{
|
||||
"type": "done" if done else "failed",
|
||||
"status": current.get("status"),
|
||||
"row_count": result.get("row_count"),
|
||||
"truncated": result.get("truncated"),
|
||||
"result_url": f"/dbapi/query/{uid}/result" if done else None,
|
||||
"error": current.get("error") or None,
|
||||
}
|
||||
)
|
||||
return
|
||||
while True:
|
||||
frame = await listener.get()
|
||||
await websocket.send_json(frame)
|
||||
if frame.get("type") in ("done", "failed"):
|
||||
break
|
||||
except WebSocketDisconnect:
|
||||
pass
|
||||
except Exception:
|
||||
logger.exception("dbquery websocket loop failed for %s", uid)
|
||||
finally:
|
||||
hub.unsubscribe(uid, listener)
|
||||
|
||||
|
||||
def _audit_query(request, caller, sql, row_count, event="database.query"):
|
||||
from devplacepy.services.audit import record as audit
|
||||
|
||||
audit.record(
|
||||
request,
|
||||
event,
|
||||
summary=f"{caller.username or caller.kind} ran a database query",
|
||||
metadata={"sql": sql[:500], "row_count": row_count, "caller": caller.kind},
|
||||
)
|
||||
@@ -0,0 +1,36 @@
|
||||
# retoor <retoor@molodetz.nl>
|
||||
|
||||
from fastapi import APIRouter, Request
|
||||
from fastapi.responses import JSONResponse
|
||||
|
||||
from devplacepy.database import get_table
|
||||
from devplacepy.schemas import DbSchemaOut, DbTableListOut
|
||||
from devplacepy.services.dbapi import crud, policy
|
||||
|
||||
from ._shared import assert_table, require_dbapi_caller
|
||||
|
||||
router = APIRouter()
|
||||
|
||||
|
||||
@router.get("/tables")
|
||||
async def dbapi_tables(request: Request):
|
||||
require_dbapi_caller(request)
|
||||
tables = []
|
||||
for name in policy.allowed_tables():
|
||||
tables.append(
|
||||
{
|
||||
"name": name,
|
||||
"row_count": get_table(name).count(),
|
||||
"soft_delete": policy.soft_delete_aware(name),
|
||||
}
|
||||
)
|
||||
return JSONResponse(
|
||||
DbTableListOut(tables=tables, count=len(tables)).model_dump(mode="json")
|
||||
)
|
||||
|
||||
|
||||
@router.get("/{table}/schema")
|
||||
async def dbapi_schema(request: Request, table: str):
|
||||
require_dbapi_caller(request)
|
||||
assert_table(table)
|
||||
return JSONResponse(DbSchemaOut(**crud.schema(table)).model_dump(mode="json"))
|
||||
Reference in New Issue
Block a user