feat: add admin/internal database API with CRUD, read-only query, and natural-language SQL endpoints

Add a new `/dbapi` router package providing a generic database API over `dataset`, restricted to admin sessions, admin API keys, and the internal gateway key. Includes:

- `tables.py`: list all tables and inspect table schemas
- `crud.py`: full CRUD operations (GET, POST, PATCH, DELETE) with soft-delete awareness, born-live inserts, `?include_deleted`, `.../restore`, and `?hard=true` purge
- `query.py`: validated read-only SELECT execution via sqlglot parsing, classification, and EXPLAIN dry-run; async query jobs with WebSocket streaming via `DbApiJobService`
- `nl.py`: natural-language-to-SQL conversion using the platform AI gateway with re-prompting until validation passes

Also register `DbApiJobService` and `PubSubService` in the service manager, add `DBAPI_DIR` to config data paths, and force cleartext `http://` connections to HTTP/1.1 in `curl_transport` to fix large request failures against uvicorn's HTTP/1.1-only internal gateway.

tests/unit/services/dbapi/__init__.py

@@ -0,0 +1 @@
+# retoor <retoor@molodetz.nl>

tests/unit/services/dbapi/conftest.py

@@ -0,0 +1,43 @@
+# retoor <retoor@molodetz.nl>
+
+from datetime import datetime, timezone
+
+import pytest
+
+from devplacepy.database import get_table, purge, refresh_snapshot
+from devplacepy.utils import generate_uid
+
+
+@pytest.fixture(autouse=True)
+def _seed_dbapi_schema(local_db):
+    now = datetime.now(timezone.utc).isoformat()
+    users = get_table("users")
+    if "username" not in users.columns:
+        users.insert(
+            {
+                "uid": generate_uid(),
+                "username": "dbapi_seed",
+                "email": "seed@dbapi.test",
+                "role": "Member",
+                "xp": 0,
+                "level": 1,
+                "created_at": now,
+            }
+        )
+    bookmarks = get_table("bookmarks")
+    if "user_uid" not in bookmarks.columns:
+        seed = generate_uid()
+        bookmarks.insert(
+            {
+                "uid": seed,
+                "user_uid": "_seedcols",
+                "target_type": "post",
+                "target_uid": "_s",
+                "created_at": now,
+                "deleted_at": None,
+                "deleted_by": None,
+            }
+        )
+        purge("bookmarks", uid=seed)
+    refresh_snapshot()
+    yield

tests/unit/services/dbapi/crud.py

@@ -0,0 +1,90 @@
+# retoor <retoor@molodetz.nl>
+
+import pytest
+
+from devplacepy.database import get_table, purge
+from devplacepy.services.dbapi import crud
+from devplacepy.services.dbapi.crud import DbApiError
+
+ACTOR = "unit_dbapi_actor"
+MARK = "_unit_dbapi_crud"
+
+
+@pytest.fixture
+def clean_bookmarks(local_db):
+    yield
+    purge("bookmarks", user_uid=MARK)
+
+
+def _insert():
+    return crud.insert_row(
+        "bookmarks",
+        {"user_uid": MARK, "target_type": "post", "target_uid": "t1"},
+        ACTOR,
+    )
+
+
+def test_insert_is_born_live(clean_bookmarks):
+    row = _insert()
+    assert row["uid"]
+    assert row["deleted_at"] is None
+    assert row["deleted_by"] is None
+    assert row.get("created_at")
+
+
+def test_insert_rejects_unknown_columns(clean_bookmarks):
+    with pytest.raises(DbApiError):
+        crud.insert_row("bookmarks", {"user_uid": MARK, "no_such_col": 1}, ACTOR)
+
+
+def test_get_and_update_row(clean_bookmarks):
+    row = _insert()
+    fetched = crud.get_row("bookmarks", "uid", row["uid"])
+    assert fetched["uid"] == row["uid"]
+    updated = crud.update_row(
+        "bookmarks", "uid", row["uid"], {"target_uid": "t2"}, ACTOR
+    )
+    assert updated["target_uid"] == "t2"
+
+
+def test_update_cannot_change_uid(clean_bookmarks):
+    row = _insert()
+    crud.update_row("bookmarks", "uid", row["uid"], {"uid": "hacked"}, ACTOR)
+    assert get_table("bookmarks").find_one(uid="hacked") is None
+
+
+def test_soft_delete_then_restore(clean_bookmarks):
+    row = _insert()
+    outcome = crud.delete_row("bookmarks", "uid", row["uid"], ACTOR)
+    assert outcome["mode"] == "soft"
+    live, _ = crud.list_rows("bookmarks", filters={"user_uid": MARK})
+    assert live == []
+    deleted, _ = crud.list_rows(
+        "bookmarks", filters={"user_uid": MARK}, include_deleted=True
+    )
+    assert len(deleted) == 1
+    restored = crud.restore_row("bookmarks", "uid", row["uid"], ACTOR)
+    assert restored["deleted_at"] is None
+
+
+def test_hard_delete_purges(clean_bookmarks):
+    row = _insert()
+    outcome = crud.delete_row("bookmarks", "uid", row["uid"], ACTOR, hard=True)
+    assert outcome["mode"] == "hard"
+    assert get_table("bookmarks").find_one(uid=row["uid"]) is None
+
+
+def test_list_rows_caps_limit(clean_bookmarks):
+    rows, _ = crud.list_rows("bookmarks", limit=99999)
+    assert isinstance(rows, list)
+
+
+def test_schema_reports_soft_delete(local_db):
+    schema = crud.schema("bookmarks")
+    assert schema["soft_delete"] is True
+    assert any(c["name"] == "uid" for c in schema["columns"])
+
+
+def test_bad_key_raises(local_db):
+    with pytest.raises(DbApiError):
+        crud.get_row("bookmarks", "no_such_key", "x")

tests/unit/services/dbapi/policy.py

@@ -0,0 +1,48 @@
+# retoor <retoor@molodetz.nl>
+
+import pytest
+
+from devplacepy.database import set_setting
+from devplacepy.services.dbapi import policy
+from devplacepy.services.dbapi.policy import DbApiBadTable
+
+
+def test_assert_table_accepts_known_table(local_db):
+    assert policy.assert_table("users") == "users"
+
+
+def test_assert_table_rejects_bad_name(local_db):
+    with pytest.raises(DbApiBadTable):
+        policy.assert_table("users; drop")
+
+
+def test_assert_table_rejects_denied(local_db):
+    with pytest.raises(DbApiBadTable):
+        policy.assert_table("sessions")
+
+
+def test_assert_table_rejects_unknown(local_db):
+    with pytest.raises(DbApiBadTable):
+        policy.assert_table("table_that_does_not_exist_zzz")
+
+
+def test_allowed_tables_excludes_deny(local_db):
+    tables = policy.allowed_tables()
+    assert "sessions" not in tables
+    assert "password_resets" not in tables
+    assert "users" in tables
+
+
+def test_soft_delete_aware():
+    assert policy.soft_delete_aware("posts") is True
+    assert policy.soft_delete_aware("users") is False
+
+
+def test_deny_tables_includes_setting(local_db):
+    set_setting("dbapi_deny_tables", "users, news")
+    try:
+        deny = policy.deny_tables()
+        assert "users" in deny and "news" in deny
+        assert "sessions" in deny
+    finally:
+        set_setting("dbapi_deny_tables", "")

tests/unit/services/dbapi/service.py

@@ -0,0 +1,32 @@
+# retoor <retoor@molodetz.nl>
+
+from tests.conftest import run_async
+
+from devplacepy.services.dbapi.service import DbApiJobService
+
+
+def test_process_runs_select_and_writes_result(local_db):
+    svc = DbApiJobService()
+    job = {"uid": "unit-dbquery-1", "payload": {"sql": "SELECT 1 AS n"}}
+    try:
+        result = run_async(svc.process(job))
+        assert result["row_count"] == 1
+        assert result["truncated"] is False
+        assert result["result_url"].endswith("/result")
+        assert (svc.result_dir("unit-dbquery-1") / "result.json").is_file()
+    finally:
+        svc.cleanup(job)
+        assert not svc.result_dir("unit-dbquery-1").exists()
+
+
+def test_process_rejects_non_select(local_db):
+    svc = DbApiJobService()
+    job = {"uid": "unit-dbquery-2", "payload": {"sql": "DELETE FROM posts"}}
+    raised = False
+    try:
+        run_async(svc.process(job))
+    except Exception:
+        raised = True
+    finally:
+        svc.cleanup(job)
+    assert raised

tests/unit/services/dbapi/validate.py

@@ -0,0 +1,84 @@
+# retoor <retoor@molodetz.nl>
+
+from devplacepy.services.dbapi import validate
+
+
+def test_classify_select_with_clauses():
+    v = validate.classify(
+        "SELECT u.uid FROM users u JOIN posts p ON p.user_uid = u.uid WHERE u.role = 'Admin' LIMIT 5"
+    )
+    assert v.statement_type == "select"
+    assert v.is_select is True
+    assert v.has_where and v.has_join and v.has_limit
+    assert v.suspicious == []
+    assert "users" in v.tables and "posts" in v.tables
+
+
+def test_classify_select_without_clauses_is_suspicious():
+    v = validate.classify("SELECT * FROM users")
+    assert v.is_select is True
+    assert any("WHERE" in note for note in v.suspicious)
+
+
+def test_classify_rejects_mutations():
+    for sql, kind in (
+        ("DELETE FROM posts", "delete"),
+        ("UPDATE users SET role = 'Admin'", "update"),
+        ("INSERT INTO posts (uid) VALUES ('x')", "insert"),
+        ("DROP TABLE users", "ddl"),
+    ):
+        v = validate.classify(sql)
+        assert v.statement_type == kind
+        assert v.is_select is False
+
+
+def test_classify_multiple_statements():
+    v = validate.classify("SELECT 1; DROP TABLE users")
+    assert any("Multiple statements" in note for note in v.suspicious)
+
+
+def test_classify_blocks_attach_pragma():
+    v = validate.classify("ATTACH DATABASE 'x.db' AS y")
+    assert v.is_select is False
+    assert any("ATTACH" in note for note in v.suspicious)
+
+
+def test_classify_parse_error_sets_error():
+    v = validate.classify("SELEKT FROM WHERE")
+    assert v.error is not None
+
+
+def test_validate_select_accepts_real_select(local_db):
+    v = validate.validate_select("SELECT uid, username FROM users LIMIT 3")
+    assert v.valid is True
+    assert v.error is None
+
+
+def test_validate_select_rejects_non_select(local_db):
+    v = validate.validate_select("DELETE FROM posts")
+    assert v.valid is False
+    assert "SELECT" in v.error
+
+
+def test_validate_select_dry_run_catches_bad_table(local_db):
+    v = validate.validate_select("SELECT * FROM no_such_table_zzz LIMIT 1")
+    assert v.valid is False
+    assert "no such table" in v.error
+
+
+def test_run_select_returns_rows_and_truncation(local_db):
+    rows, truncated = validate.run_select("SELECT 1 AS n", 10)
+    assert rows == [{"n": 1}]
+    assert truncated is False
+
+
+def test_run_select_truncates(local_db):
+    rows, truncated = validate.run_select(
+        "SELECT 1 AS n UNION ALL SELECT 2 UNION ALL SELECT 3", 2
+    )
+    assert len(rows) == 2
+    assert truncated is True
+
+
+def test_transpile_returns_sql():
+    assert "SELECT" in validate.transpile("SELECT 1").upper()