feat: add admin/internal database API with CRUD, read-only query, and natural-language SQL endpoints

Add a new `/dbapi` router package providing a generic database API over `dataset`, restricted to admin sessions, admin API keys, and the internal gateway key. Includes:

- `tables.py`: list all tables and inspect table schemas
- `crud.py`: full CRUD operations (GET, POST, PATCH, DELETE) with soft-delete awareness, born-live inserts, `?include_deleted`, `.../restore`, and `?hard=true` purge
- `query.py`: validated read-only SELECT execution via sqlglot parsing, classification, and EXPLAIN dry-run; async query jobs with WebSocket streaming via `DbApiJobService`
- `nl.py`: natural-language-to-SQL conversion using the platform AI gateway with re-prompting until validation passes

Also register `DbApiJobService` and `PubSubService` in the service manager, add `DBAPI_DIR` to config data paths, and force cleartext `http://` connections to HTTP/1.1 in `curl_transport` to fix large request failures against uvicorn's HTTP/1.1-only internal gateway.

tests/api/dbapi/__init__.py

@@ -0,0 +1 @@
+# retoor <retoor@molodetz.nl>

tests/api/dbapi/conftest.py

@@ -0,0 +1,58 @@
+# retoor <retoor@molodetz.nl>
+
+from datetime import datetime, timezone
+
+import pytest
+from fastapi.testclient import TestClient
+
+from devplacepy.database import get_table, internal_gateway_key, purge, refresh_snapshot
+from devplacepy.main import app
+from devplacepy.services.manager import service_manager
+from devplacepy.utils import generate_uid
+
+
+def _seed_schema():
+    now = datetime.now(timezone.utc).isoformat()
+    users = get_table("users")
+    if "username" not in users.columns:
+        users.insert(
+            {
+                "uid": generate_uid(),
+                "username": "dbapi_seed",
+                "email": "seed@dbapi.test",
+                "role": "Member",
+                "xp": 0,
+                "level": 1,
+                "created_at": now,
+            }
+        )
+    bookmarks = get_table("bookmarks")
+    if "user_uid" not in bookmarks.columns:
+        seed = generate_uid()
+        bookmarks.insert(
+            {
+                "uid": seed,
+                "user_uid": "_seedcols",
+                "target_type": "post",
+                "target_uid": "_s",
+                "created_at": now,
+                "deleted_at": None,
+                "deleted_by": None,
+            }
+        )
+        purge("bookmarks", uid=seed)
+    refresh_snapshot()
+
+
+@pytest.fixture(scope="module")
+def client():
+    with TestClient(app) as test_client:
+        service_manager.set_lock_owner(True)
+        _seed_schema()
+        yield test_client
+        service_manager.set_lock_owner(False)
+
+
+@pytest.fixture
+def auth(client):
+    return {"X-API-KEY": internal_gateway_key()}

tests/api/dbapi/index.py

@@ -0,0 +1,52 @@
+# retoor <retoor@molodetz.nl>
+
+MARK = "_apitest_dbapi_crud"
+
+
+def test_list_requires_auth(client):
+    assert client.get("/dbapi/users").status_code == 403
+
+
+def test_crud_round_trip(client, auth):
+    created = client.post(
+        "/dbapi/bookmarks",
+        json={"user_uid": MARK, "target_type": "post", "target_uid": "t1"},
+        headers=auth,
+    )
+    assert created.status_code == 200
+    row = created.json()["row"]
+    uid = row["uid"]
+    assert row["deleted_at"] is None
+    try:
+        got = client.get(f"/dbapi/bookmarks/uid/{uid}", headers=auth)
+        assert got.json()["row"]["uid"] == uid
+
+        updated = client.patch(
+            f"/dbapi/bookmarks/uid/{uid}", json={"target_uid": "t2"}, headers=auth
+        )
+        assert updated.json()["row"]["target_uid"] == "t2"
+
+        deleted = client.delete(f"/dbapi/bookmarks/uid/{uid}", headers=auth)
+        assert deleted.json()["mode"] == "soft"
+        live = client.get(f"/dbapi/bookmarks?filter.user_uid={MARK}", headers=auth)
+        assert live.json()["count"] == 0
+        with_deleted = client.get(
+            f"/dbapi/bookmarks?filter.user_uid={MARK}&include_deleted=true", headers=auth
+        )
+        assert with_deleted.json()["count"] == 1
+
+        restored = client.post(f"/dbapi/bookmarks/uid/{uid}/restore", headers=auth)
+        assert restored.json()["row"]["deleted_at"] is None
+    finally:
+        client.delete(f"/dbapi/bookmarks/uid/{uid}?hard=true", headers=auth)
+
+
+def test_insert_unknown_column_is_400(client, auth):
+    response = client.post(
+        "/dbapi/bookmarks", json={"user_uid": MARK, "no_such_col": 1}, headers=auth
+    )
+    assert response.status_code == 400
+
+
+def test_get_missing_row_is_404(client, auth):
+    assert client.get("/dbapi/bookmarks/uid/does-not-exist", headers=auth).status_code == 404

tests/api/dbapi/nl.py

@@ -0,0 +1,37 @@
+# retoor <retoor@molodetz.nl>
+
+
+def test_nl_requires_auth(client):
+    assert (
+        client.post("/dbapi/nl", json={"question": "x", "table": "users"}).status_code
+        == 403
+    )
+
+
+def test_nl_missing_fields_is_400(client, auth):
+    assert client.post("/dbapi/nl", json={"question": "x"}, headers=auth).status_code == 400
+
+
+def test_nl_denied_table_is_404(client, auth):
+    response = client.post(
+        "/dbapi/nl", json={"question": "x", "table": "sessions"}, headers=auth
+    )
+    assert response.status_code == 404
+
+
+def test_nl_is_graceful_when_gateway_fails(client, auth, monkeypatch):
+    async def _boom(*args, **kwargs):
+        raise RuntimeError("gateway down")
+
+    monkeypatch.setattr(
+        "devplacepy.services.dbapi.nl2sql._complete", _boom
+    )
+    response = client.post(
+        "/dbapi/nl",
+        json={"question": "all administrators", "table": "users"},
+        headers=auth,
+    )
+    assert response.status_code == 200
+    body = response.json()
+    assert body["valid"] is False
+    assert body["error"]

tests/api/dbapi/query.py

@@ -0,0 +1,76 @@
+# retoor <retoor@molodetz.nl>
+
+import pytest
+from starlette.websockets import WebSocketDisconnect
+
+from devplacepy.services.manager import service_manager
+
+
+def test_query_requires_auth(client):
+    assert client.post("/dbapi/query", json={"sql": "SELECT 1"}).status_code == 403
+
+
+def test_select_runs(client, auth):
+    response = client.post("/dbapi/query", json={"sql": "SELECT 1 AS n"}, headers=auth)
+    assert response.status_code == 200
+    body = response.json()
+    assert body["valid"] is True
+    assert body["rows"] == [{"n": 1}]
+
+
+def test_non_select_is_409(client, auth):
+    response = client.post(
+        "/dbapi/query", json={"sql": "DELETE FROM posts"}, headers=auth
+    )
+    assert response.status_code == 409
+    assert response.json()["statement_type"] == "delete"
+
+
+def test_invalid_sql_is_400(client, auth):
+    response = client.post(
+        "/dbapi/query", json={"sql": "SELECT * FROM no_such_table_zzz"}, headers=auth
+    )
+    assert response.status_code == 400
+
+
+def test_suspicious_select_is_flagged(client, auth):
+    response = client.post(
+        "/dbapi/query", json={"sql": "SELECT * FROM users"}, headers=auth
+    )
+    assert response.status_code == 200
+    assert response.json()["suspicious"]
+
+
+def test_async_enqueue_and_status(client, auth):
+    enqueued = client.post(
+        "/dbapi/query/async", json={"sql": "SELECT uid FROM users LIMIT 3"}, headers=auth
+    )
+    assert enqueued.status_code == 200
+    uid = enqueued.json()["uid"]
+    assert enqueued.json()["ws_url"].endswith("/ws")
+    assert client.get(f"/dbapi/query/{uid}", headers=auth).status_code == 200
+
+
+def test_async_non_select_is_409(client, auth):
+    response = client.post(
+        "/dbapi/query/async", json={"sql": "UPDATE users SET role = 1"}, headers=auth
+    )
+    assert response.status_code == 409
+
+
+def test_ws_unknown_job_is_closed(client, auth):
+    with client.websocket_connect("/dbapi/query/not-a-real-uid/ws") as ws:
+        with pytest.raises(WebSocketDisconnect) as info:
+            ws.receive_json()
+    assert info.value.code == 1008
+
+
+def test_ws_non_owner_retries(client, auth):
+    service_manager.set_lock_owner(False)
+    try:
+        with client.websocket_connect("/dbapi/query/anything/ws") as ws:
+            with pytest.raises(WebSocketDisconnect) as info:
+                ws.receive_json()
+        assert info.value.code == 4013
+    finally:
+        service_manager.set_lock_owner(True)

tests/api/dbapi/tables.py

@@ -0,0 +1,24 @@
+# retoor <retoor@molodetz.nl>
+
+
+def test_tables_requires_auth(client):
+    assert client.get("/dbapi/tables").status_code == 403
+
+
+def test_tables_lists_and_hides_denied(client, auth):
+    response = client.get("/dbapi/tables", headers=auth)
+    assert response.status_code == 200
+    names = {row["name"] for row in response.json()["tables"]}
+    assert "users" in names
+    assert "sessions" not in names
+    assert "password_resets" not in names
+
+
+def test_schema(client, auth):
+    response = client.get("/dbapi/users/schema", headers=auth)
+    assert response.status_code == 200
+    assert any(column["name"] == "uid" for column in response.json()["columns"])
+
+
+def test_schema_denied_table_is_404(client, auth):
+    assert client.get("/dbapi/sessions/schema", headers=auth).status_code == 404

tests/api/pubsub/__init__.py

@@ -0,0 +1 @@
+# retoor <retoor@molodetz.nl>

tests/api/pubsub/conftest.py

@@ -0,0 +1,22 @@
+# retoor <retoor@molodetz.nl>
+
+import pytest
+from fastapi.testclient import TestClient
+
+from devplacepy.database import internal_gateway_key, set_setting
+from devplacepy.main import app
+from devplacepy.services.manager import service_manager
+
+
+@pytest.fixture(scope="module")
+def client():
+    set_setting("pubsub_allow_guests", "0")
+    with TestClient(app) as test_client:
+        service_manager.set_lock_owner(True)
+        yield test_client
+        service_manager.set_lock_owner(False)
+
+
+@pytest.fixture
+def auth(client):
+    return {"X-API-KEY": internal_gateway_key()}

tests/api/pubsub/index.py

@@ -0,0 +1,67 @@
+# retoor <retoor@molodetz.nl>
+
+import pytest
+from starlette.websockets import WebSocketDisconnect
+
+from devplacepy.services.manager import service_manager
+
+
+def test_publish_requires_auth(client):
+    assert client.post("/pubsub/publish", json={"topic": "public.x", "data": 1}).status_code == 403
+
+
+def test_topics_requires_auth(client):
+    assert client.get("/pubsub/topics").status_code == 403
+
+
+def test_http_publish_and_topics(client, auth):
+    assert (
+        client.post(
+            "/pubsub/publish", json={"topic": "public.demo", "data": {"a": 1}}, headers=auth
+        ).status_code
+        == 200
+    )
+    assert client.get("/pubsub/topics", headers=auth).status_code == 200
+
+
+def test_ws_pubsub_delivery(client, auth):
+    with client.websocket_connect("/pubsub/ws", headers=auth) as receiver:
+        assert receiver.receive_json()["type"] == "ready"
+        receiver.send_json({"type": "subscribe", "topic": "public.room"})
+        assert receiver.receive_json()["type"] == "subscribed"
+        with client.websocket_connect("/pubsub/ws", headers=auth) as sender:
+            sender.receive_json()
+            sender.send_json(
+                {"type": "publish", "topic": "public.room", "data": {"hi": 1}}
+            )
+            ack = sender.receive_json()
+            assert ack["type"] == "ack" and ack["delivered"] == 1
+            message = receiver.receive_json()
+            assert message["type"] == "message"
+            assert message["topic"] == "public.room"
+            assert message["data"] == {"hi": 1}
+
+
+def test_ws_invalid_topic_errors(client, auth):
+    with client.websocket_connect("/pubsub/ws", headers=auth) as ws:
+        ws.receive_json()
+        ws.send_json({"type": "subscribe", "topic": "bad topic"})
+        assert ws.receive_json()["type"] == "error"
+
+
+def test_ws_guest_closed_when_disabled(client):
+    with client.websocket_connect("/pubsub/ws") as ws:
+        with pytest.raises(WebSocketDisconnect) as info:
+            ws.receive_json()
+    assert info.value.code == 1008
+
+
+def test_ws_non_owner_retries(client, auth):
+    service_manager.set_lock_owner(False)
+    try:
+        with client.websocket_connect("/pubsub/ws", headers=auth) as ws:
+            with pytest.raises(WebSocketDisconnect) as info:
+                ws.receive_json()
+        assert info.value.code == 4013
+    finally:
+        service_manager.set_lock_owner(True)