feat: add admin/internal database API with CRUD, read-only query, and natural-language SQL endpoints

Add a new `/dbapi` router package providing a generic database API over `dataset`, restricted to admin sessions, admin API keys, and the internal gateway key. Includes: - `tables.py`: list all tables and inspect table schemas - `crud.py`: full CRUD operations (GET, POST, PATCH, DELETE) with soft-delete awareness, born-live inserts, `?include_deleted`, `.../restore`, and `?hard=true` purge - `query.py`: validated read-only SELECT execution via sqlglot parsing, classification, and EXPLAIN dry-run; async query jobs with WebSocket streaming via `DbApiJobService` - `nl.py`: natural-language-to-SQL conversion using the platform AI gateway with re-prompting until validation passes Also register `DbApiJobService` and `PubSubService` in the service manager, add `DBAPI_DIR` to config data paths, and force cleartext `http://` connections to HTTP/1.1 in `curl_transport` to fix large request failures against uvicorn's HTTP/1.1-only internal gateway.
2026-06-14 23:00:30 +00:00
parent 96521e8757
commit d31ccba6c1
69 changed files with 3278 additions and 12 deletions
@@ -145,6 +145,34 @@ def test_reconcile_launches_and_stops(env):
    assert store.get_instance(inst["uid"])["status"] == store.ST_STOPPED


+def test_manual_start_relaunches_after_never_policy_exit(env):
+    fake = env["fake"]
+    inst = _ready_instance(env, restart_policy="never", autostart=True)
+    service = ContainerService()
+    run_async(service.run_once())
+    refresh_snapshot()
+    inst = store.get_instance(inst["uid"])
+    cid = inst["container_id"]
+    assert inst["status"] == store.ST_RUNNING and cid
+
+    fake._set_state(cid, "exited", 0)
+    run_async(service.run_once())
+    refresh_snapshot()
+    inst = store.get_instance(inst["uid"])
+    assert inst["status"] == store.ST_STOPPED
+    assert inst["desired_state"] == store.DESIRED_STOPPED
+
+    api.set_desired_state(inst, store.DESIRED_RUNNING)
+    refresh_snapshot()
+    run_async(service.run_once())
+    refresh_snapshot()
+    inst = store.get_instance(inst["uid"])
+    assert inst["status"] == store.ST_RUNNING
+    assert inst["desired_state"] == store.DESIRED_RUNNING
+    assert cid in fake.removed
+    assert inst["container_id"] and inst["container_id"] != cid
+
+
 def test_reconcile_reaps_orphan(env):
    fake = env["fake"]
    run_async(
@@ -0,0 +1 @@
+# retoor <retoor@molodetz.nl>
@@ -0,0 +1,43 @@
+# retoor <retoor@molodetz.nl>
+
+from datetime import datetime, timezone
+
+import pytest
+
+from devplacepy.database import get_table, purge, refresh_snapshot
+from devplacepy.utils import generate_uid
+
+
+@pytest.fixture(autouse=True)
+def _seed_dbapi_schema(local_db):
+    now = datetime.now(timezone.utc).isoformat()
+    users = get_table("users")
+    if "username" not in users.columns:
+        users.insert(
+            {
+                "uid": generate_uid(),
+                "username": "dbapi_seed",
+                "email": "seed@dbapi.test",
+                "role": "Member",
+                "xp": 0,
+                "level": 1,
+                "created_at": now,
+            }
+        )
+    bookmarks = get_table("bookmarks")
+    if "user_uid" not in bookmarks.columns:
+        seed = generate_uid()
+        bookmarks.insert(
+            {
+                "uid": seed,
+                "user_uid": "_seedcols",
+                "target_type": "post",
+                "target_uid": "_s",
+                "created_at": now,
+                "deleted_at": None,
+                "deleted_by": None,
+            }
+        )
+        purge("bookmarks", uid=seed)
+    refresh_snapshot()
+    yield
@@ -0,0 +1,90 @@
+# retoor <retoor@molodetz.nl>
+
+import pytest
+
+from devplacepy.database import get_table, purge
+from devplacepy.services.dbapi import crud
+from devplacepy.services.dbapi.crud import DbApiError
+
+ACTOR = "unit_dbapi_actor"
+MARK = "_unit_dbapi_crud"
+
+
+@pytest.fixture
+def clean_bookmarks(local_db):
+    yield
+    purge("bookmarks", user_uid=MARK)
+
+
+def _insert():
+    return crud.insert_row(
+        "bookmarks",
+        {"user_uid": MARK, "target_type": "post", "target_uid": "t1"},
+        ACTOR,
+    )
+
+
+def test_insert_is_born_live(clean_bookmarks):
+    row = _insert()
+    assert row["uid"]
+    assert row["deleted_at"] is None
+    assert row["deleted_by"] is None
+    assert row.get("created_at")
+
+
+def test_insert_rejects_unknown_columns(clean_bookmarks):
+    with pytest.raises(DbApiError):
+        crud.insert_row("bookmarks", {"user_uid": MARK, "no_such_col": 1}, ACTOR)
+
+
+def test_get_and_update_row(clean_bookmarks):
+    row = _insert()
+    fetched = crud.get_row("bookmarks", "uid", row["uid"])
+    assert fetched["uid"] == row["uid"]
+    updated = crud.update_row(
+        "bookmarks", "uid", row["uid"], {"target_uid": "t2"}, ACTOR
+    )
+    assert updated["target_uid"] == "t2"
+
+
+def test_update_cannot_change_uid(clean_bookmarks):
+    row = _insert()
+    crud.update_row("bookmarks", "uid", row["uid"], {"uid": "hacked"}, ACTOR)
+    assert get_table("bookmarks").find_one(uid="hacked") is None
+
+
+def test_soft_delete_then_restore(clean_bookmarks):
+    row = _insert()
+    outcome = crud.delete_row("bookmarks", "uid", row["uid"], ACTOR)
+    assert outcome["mode"] == "soft"
+    live, _ = crud.list_rows("bookmarks", filters={"user_uid": MARK})
+    assert live == []
+    deleted, _ = crud.list_rows(
+        "bookmarks", filters={"user_uid": MARK}, include_deleted=True
+    )
+    assert len(deleted) == 1
+    restored = crud.restore_row("bookmarks", "uid", row["uid"], ACTOR)
+    assert restored["deleted_at"] is None
+
+
+def test_hard_delete_purges(clean_bookmarks):
+    row = _insert()
+    outcome = crud.delete_row("bookmarks", "uid", row["uid"], ACTOR, hard=True)
+    assert outcome["mode"] == "hard"
+    assert get_table("bookmarks").find_one(uid=row["uid"]) is None
+
+
+def test_list_rows_caps_limit(clean_bookmarks):
+    rows, _ = crud.list_rows("bookmarks", limit=99999)
+    assert isinstance(rows, list)
+
+
+def test_schema_reports_soft_delete(local_db):
+    schema = crud.schema("bookmarks")
+    assert schema["soft_delete"] is True
+    assert any(c["name"] == "uid" for c in schema["columns"])
+
+
+def test_bad_key_raises(local_db):
+    with pytest.raises(DbApiError):
+        crud.get_row("bookmarks", "no_such_key", "x")
@@ -0,0 +1,48 @@
+# retoor <retoor@molodetz.nl>
+
+import pytest
+
+from devplacepy.database import set_setting
+from devplacepy.services.dbapi import policy
+from devplacepy.services.dbapi.policy import DbApiBadTable
+
+
+def test_assert_table_accepts_known_table(local_db):
+    assert policy.assert_table("users") == "users"
+
+
+def test_assert_table_rejects_bad_name(local_db):
+    with pytest.raises(DbApiBadTable):
+        policy.assert_table("users; drop")
+
+
+def test_assert_table_rejects_denied(local_db):
+    with pytest.raises(DbApiBadTable):
+        policy.assert_table("sessions")
+
+
+def test_assert_table_rejects_unknown(local_db):
+    with pytest.raises(DbApiBadTable):
+        policy.assert_table("table_that_does_not_exist_zzz")
+
+
+def test_allowed_tables_excludes_deny(local_db):
+    tables = policy.allowed_tables()
+    assert "sessions" not in tables
+    assert "password_resets" not in tables
+    assert "users" in tables
+
+
+def test_soft_delete_aware():
+    assert policy.soft_delete_aware("posts") is True
+    assert policy.soft_delete_aware("users") is False
+
+
+def test_deny_tables_includes_setting(local_db):
+    set_setting("dbapi_deny_tables", "users, news")
+    try:
+        deny = policy.deny_tables()
+        assert "users" in deny and "news" in deny
+        assert "sessions" in deny
+    finally:
+        set_setting("dbapi_deny_tables", "")
@@ -0,0 +1,32 @@
+# retoor <retoor@molodetz.nl>
+
+from tests.conftest import run_async
+
+from devplacepy.services.dbapi.service import DbApiJobService
+
+
+def test_process_runs_select_and_writes_result(local_db):
+    svc = DbApiJobService()
+    job = {"uid": "unit-dbquery-1", "payload": {"sql": "SELECT 1 AS n"}}
+    try:
+        result = run_async(svc.process(job))
+        assert result["row_count"] == 1
+        assert result["truncated"] is False
+        assert result["result_url"].endswith("/result")
+        assert (svc.result_dir("unit-dbquery-1") / "result.json").is_file()
+    finally:
+        svc.cleanup(job)
+        assert not svc.result_dir("unit-dbquery-1").exists()
+
+
+def test_process_rejects_non_select(local_db):
+    svc = DbApiJobService()
+    job = {"uid": "unit-dbquery-2", "payload": {"sql": "DELETE FROM posts"}}
+    raised = False
+    try:
+        run_async(svc.process(job))
+    except Exception:
+        raised = True
+    finally:
+        svc.cleanup(job)
+    assert raised
@@ -0,0 +1,84 @@
+# retoor <retoor@molodetz.nl>
+
+from devplacepy.services.dbapi import validate
+
+
+def test_classify_select_with_clauses():
+    v = validate.classify(
+        "SELECT u.uid FROM users u JOIN posts p ON p.user_uid = u.uid WHERE u.role = 'Admin' LIMIT 5"
+    )
+    assert v.statement_type == "select"
+    assert v.is_select is True
+    assert v.has_where and v.has_join and v.has_limit
+    assert v.suspicious == []
+    assert "users" in v.tables and "posts" in v.tables
+
+
+def test_classify_select_without_clauses_is_suspicious():
+    v = validate.classify("SELECT * FROM users")
+    assert v.is_select is True
+    assert any("WHERE" in note for note in v.suspicious)
+
+
+def test_classify_rejects_mutations():
+    for sql, kind in (
+        ("DELETE FROM posts", "delete"),
+        ("UPDATE users SET role = 'Admin'", "update"),
+        ("INSERT INTO posts (uid) VALUES ('x')", "insert"),
+        ("DROP TABLE users", "ddl"),
+    ):
+        v = validate.classify(sql)
+        assert v.statement_type == kind
+        assert v.is_select is False
+
+
+def test_classify_multiple_statements():
+    v = validate.classify("SELECT 1; DROP TABLE users")
+    assert any("Multiple statements" in note for note in v.suspicious)
+
+
+def test_classify_blocks_attach_pragma():
+    v = validate.classify("ATTACH DATABASE 'x.db' AS y")
+    assert v.is_select is False
+    assert any("ATTACH" in note for note in v.suspicious)
+
+
+def test_classify_parse_error_sets_error():
+    v = validate.classify("SELEKT FROM WHERE")
+    assert v.error is not None
+
+
+def test_validate_select_accepts_real_select(local_db):
+    v = validate.validate_select("SELECT uid, username FROM users LIMIT 3")
+    assert v.valid is True
+    assert v.error is None
+
+
+def test_validate_select_rejects_non_select(local_db):
+    v = validate.validate_select("DELETE FROM posts")
+    assert v.valid is False
+    assert "SELECT" in v.error
+
+
+def test_validate_select_dry_run_catches_bad_table(local_db):
+    v = validate.validate_select("SELECT * FROM no_such_table_zzz LIMIT 1")
+    assert v.valid is False
+    assert "no such table" in v.error
+
+
+def test_run_select_returns_rows_and_truncation(local_db):
+    rows, truncated = validate.run_select("SELECT 1 AS n", 10)
+    assert rows == [{"n": 1}]
+    assert truncated is False
+
+
+def test_run_select_truncates(local_db):
+    rows, truncated = validate.run_select(
+        "SELECT 1 AS n UNION ALL SELECT 2 UNION ALL SELECT 3", 2
+    )
+    assert len(rows) == 2
+    assert truncated is True
+
+
+def test_transpile_returns_sql():
+    assert "SELECT" in validate.transpile("SELECT 1").upper()
@@ -0,0 +1 @@
+# retoor <retoor@molodetz.nl>
@@ -0,0 +1,37 @@
+# retoor <retoor@molodetz.nl>
+
+from devplacepy.services.devii.actions.catalog import PLATFORM_CATALOG
+from devplacepy.services.devii.actions.dispatcher import (
+    CONFIRM_REQUIRED,
+    MUTATING_METHODS,
+)
+
+BY_NAME = PLATFORM_CATALOG.by_name()
+
+
+def _records_mutation(name: str) -> bool:
+    action = BY_NAME[name]
+    return action.method in MUTATING_METHODS and not action.is_read_only
+
+
+def test_read_only_query_tools_are_marked_read_only():
+    # These are POST routes (they carry a body) but only read data. They MUST be
+    # read-only so they do not trip the agentic plan gate or the verification gate
+    # and drop the rows the user asked for.
+    for name in ("db_query", "db_design_query"):
+        assert BY_NAME[name].is_read_only is True
+        assert _records_mutation(name) is False
+
+
+def test_db_get_tools_are_read_only():
+    for name in ("db_list_tables", "db_table_schema", "db_list_rows", "db_get_row"):
+        assert BY_NAME[name].is_read_only is True
+        assert _records_mutation(name) is False
+
+
+def test_db_mutation_tools_record_and_confirm():
+    for name in ("db_insert_row", "db_update_row", "db_delete_row"):
+        action = BY_NAME[name]
+        assert action.is_read_only is False
+        assert _records_mutation(name) is True
+        assert name in CONFIRM_REQUIRED
@@ -0,0 +1 @@
+# retoor <retoor@molodetz.nl>
@@ -0,0 +1,79 @@
+# retoor <retoor@molodetz.nl>
+
+from tests.conftest import run_async
+
+from devplacepy.services.pubsub.hub import PubSubHub, topic_matches
+
+
+class FakeSocket:
+    def __init__(self):
+        self.frames = []
+
+    async def send_json(self, frame):
+        self.frames.append(frame)
+
+
+class DeadSocket:
+    async def send_json(self, frame):
+        raise RuntimeError("socket closed")
+
+
+def test_topic_matches():
+    assert topic_matches("foo", "foo")
+    assert topic_matches("*", "anything.here")
+    assert topic_matches("foo.*", "foo.bar")
+    assert topic_matches("foo.*", "foo")
+    assert not topic_matches("foo.*", "fob")
+    assert not topic_matches("foo", "bar")
+
+
+def test_publish_delivers_only_to_matching():
+    hub = PubSubHub()
+    a, b = FakeSocket(), FakeSocket()
+    hub.subscribe("public.x", a)
+    hub.subscribe("public.y", b)
+    delivered = run_async(hub.publish("public.x", {"n": 1}))
+    assert delivered == 1
+    assert a.frames == [{"n": 1}]
+    assert b.frames == []
+
+
+def test_wildcard_delivery():
+    hub = PubSubHub()
+    a = FakeSocket()
+    hub.subscribe("foo.*", a)
+    run_async(hub.publish("foo.bar", {"k": 1}))
+    run_async(hub.publish("foo", {"k": 2}))
+    run_async(hub.publish("fob", {"k": 3}))
+    assert a.frames == [{"k": 1}, {"k": 2}]
+
+
+def test_unsubscribe_stops_delivery():
+    hub = PubSubHub()
+    a = FakeSocket()
+    hub.subscribe("t", a)
+    hub.unsubscribe("t", a)
+    assert run_async(hub.publish("t", {})) == 0
+
+
+def test_drop_socket_removes_all_subscriptions():
+    hub = PubSubHub()
+    a = FakeSocket()
+    hub.subscribe("a", a)
+    hub.subscribe("b", a)
+    hub.drop_socket(a)
+    assert hub.topics() == []
+
+
+def test_dead_socket_is_dropped_silently():
+    hub = PubSubHub()
+    hub.subscribe("t", DeadSocket())
+    assert run_async(hub.publish("t", {})) == 0
+
+
+def test_topics_lists_subscriber_counts():
+    hub = PubSubHub()
+    a, b = FakeSocket(), FakeSocket()
+    hub.subscribe("x", a)
+    hub.subscribe("x", b)
+    assert hub.topics() == [{"topic": "x", "subscribers": 2}]
@@ -0,0 +1,45 @@
+# retoor <retoor@molodetz.nl>
+
+from devplacepy.database import set_setting
+from devplacepy.services.pubsub import policy
+from devplacepy.services.pubsub.policy import Actor
+
+ADMIN = Actor("admin", "a", "adm")
+INTERNAL = Actor("internal", "internal", "internal")
+USER = Actor("user", "u1", "bob")
+GUEST = Actor("guest", "", "guest")
+
+
+def test_valid_topic():
+    assert policy.valid_topic("public.demo")
+    assert policy.valid_topic("user.u1.inbox")
+    assert not policy.valid_topic("bad topic")
+    assert not policy.valid_topic("")
+
+
+def test_privileged_can_use_any_topic():
+    for actor in (ADMIN, INTERNAL):
+        assert policy.can_subscribe(actor, "anything.here")
+        assert policy.can_publish(actor, "public.x")
+        assert policy.can_subscribe(actor, "*")
+
+
+def test_user_namespace_rules():
+    assert policy.can_publish(USER, "user.u1.x")
+    assert not policy.can_publish(USER, "user.u2.x")
+    assert not policy.can_publish(USER, "public.x")
+    assert policy.can_subscribe(USER, "public.x")
+    assert policy.can_subscribe(USER, "user.u1.feed")
+    assert not policy.can_subscribe(USER, "user.u2.feed")
+    assert not policy.can_subscribe(USER, "*")
+
+
+def test_guest_gated_by_setting(local_db):
+    set_setting("pubsub_allow_guests", "0")
+    try:
+        assert not policy.can_subscribe(GUEST, "public.x")
+        set_setting("pubsub_allow_guests", "1")
+        assert policy.can_subscribe(GUEST, "public.x")
+        assert not policy.can_publish(GUEST, "public.x")
+    finally:
+        set_setting("pubsub_allow_guests", "0")