Quiiz system

tests/unit/services/devii/tasks/guards.py

@@ -0,0 +1,95 @@
+# retoor <retoor@molodetz.nl>
+
+from datetime import timedelta
+
+from devplacepy.database import invalidate_admins_cache
+from devplacepy.services.devii.tasks.guards import (
+    REASON_BUDGET,
+    REASON_EXPIRED,
+    REASON_FAILURES,
+    REASON_MAX_RUNS,
+    REASON_NOT_ADMIN,
+    automation_allowed,
+    refusal,
+)
+from devplacepy.services.devii.tasks.schedule import now_utc, to_iso
+from devplacepy.utils import generate_uid
+
+
+def _account(local_db, role):
+    uid = generate_uid()
+    local_db["users"].insert(
+        {
+            "uid": uid,
+            "username": f"guard-{uid[-10:]}",
+            "role": role,
+            "deleted_at": None,
+        }
+    )
+    invalidate_admins_cache()
+    return uid
+
+
+def _row(owner_uid, **overrides):
+    row = {
+        "uid": generate_uid(),
+        "owner_kind": "user",
+        "owner_id": owner_uid,
+        "run_count": 0,
+        "created_at": to_iso(now_utc()),
+    }
+    row.update(overrides)
+    return row
+
+
+def test_only_administrators_may_automate(local_db):
+    admin = _account(local_db, "Admin")
+    member = _account(local_db, "Member")
+    assert automation_allowed("user", admin) is True
+    assert automation_allowed("user", member) is False
+    assert automation_allowed("guest", "guest-cookie") is False
+    assert automation_allowed("user", "") is False
+
+
+def test_member_owned_task_is_refused(local_db):
+    member = _account(local_db, "Member")
+    assert refusal(_row(member), now_utc()) == REASON_NOT_ADMIN
+
+
+def test_clean_admin_task_is_allowed(local_db):
+    admin = _account(local_db, "Admin")
+    assert refusal(_row(admin), now_utc()) is None
+
+
+def test_exhausted_task_is_refused(local_db):
+    admin = _account(local_db, "Admin")
+    row = _row(admin, max_runs=5, run_count=5)
+    assert refusal(row, now_utc()) == REASON_MAX_RUNS
+
+
+def test_expired_task_is_refused(local_db):
+    admin = _account(local_db, "Admin")
+    now = now_utc()
+    row = _row(admin, expires_at=to_iso(now - timedelta(minutes=1)))
+    assert refusal(row, now) == REASON_EXPIRED
+
+
+def test_task_without_an_expiry_stops_at_the_fallback_ceiling(local_db):
+    admin = _account(local_db, "Admin")
+    now = now_utc()
+    row = _row(admin, created_at=to_iso(now - timedelta(days=40)))
+    assert refusal(row, now) == REASON_EXPIRED
+
+
+def test_repeated_failures_refuse_the_task(local_db):
+    admin = _account(local_db, "Admin")
+    row = _row(admin, failure_count=3)
+    assert refusal(row, now_utc(), max_failures=3) == REASON_FAILURES
+    assert refusal(row, now_utc(), max_failures=0) is None
+
+
+def test_budget_probe_refuses_the_task(local_db):
+    admin = _account(local_db, "Admin")
+    row = _row(admin)
+    assert refusal(row, now_utc(), budget_exceeded=lambda k, i: True) == REASON_BUDGET
+    assert refusal(row, now_utc(), budget_exceeded=lambda k, i: False) is None

tests/unit/services/devii/tasks/schedule.py

@@ -0,0 +1,79 @@
+# retoor <retoor@molodetz.nl>
+
+import pytest
+
+from devplacepy.services.devii.tasks.guards import task_columns
+from devplacepy.services.devii.tasks.schedule import (
+    DEFAULT_MAX_RUNS,
+    MAX_LIFETIME_DAYS,
+    MAX_MAX_RUNS,
+    MIN_INTERVAL_SECONDS,
+    Schedule,
+    from_iso,
+    now_utc,
+)
+
+
+def test_interval_below_the_floor_is_rejected():
+    with pytest.raises(ValueError):
+        Schedule(kind="interval", every_seconds=MIN_INTERVAL_SECONDS - 1)
+    with pytest.raises(ValueError):
+        Schedule(kind="interval", every_seconds=180)
+
+
+def test_interval_at_the_floor_is_accepted():
+    schedule = Schedule(kind="interval", every_seconds=MIN_INTERVAL_SECONDS)
+    assert schedule.every_seconds == MIN_INTERVAL_SECONDS
+
+
+def test_cron_firing_faster_than_the_floor_is_rejected():
+    for expression in ("* * * * *", "*/5 * * * *", "*/14 * * * *"):
+        with pytest.raises(ValueError):
+            Schedule(kind="cron", cron=expression)
+
+
+def test_cron_with_an_uneven_step_is_rejected_for_its_boundary_hop():
+    with pytest.raises(ValueError):
+        Schedule(kind="cron", cron="*/18 * * * *")
+
+
+def test_cron_at_or_above_the_floor_is_accepted():
+    for expression in ("*/15 * * * *", "0 * * * *", "0 22 * * *"):
+        assert Schedule(kind="cron", cron=expression).cron == expression
+
+
+def test_max_runs_ceiling_is_enforced():
+    with pytest.raises(ValueError):
+        Schedule(kind="interval", every_seconds=900, max_runs=MAX_MAX_RUNS + 1)
+    with pytest.raises(ValueError):
+        Schedule(kind="interval", every_seconds=900, max_runs=0)
+
+
+def test_recurring_task_without_max_runs_gets_the_default():
+    reference = now_utc()
+    columns = task_columns(Schedule(kind="interval", every_seconds=900), reference)
+    assert columns["max_runs"] == DEFAULT_MAX_RUNS
+
+
+def test_one_shot_task_keeps_no_run_limit():
+    reference = now_utc()
+    columns = task_columns(Schedule(kind="once", delay_seconds=60), reference)
+    assert columns["max_runs"] is None
+
+
+def test_every_task_gets_an_expiry_within_the_ceiling():
+    reference = now_utc()
+    columns = task_columns(Schedule(kind="interval", every_seconds=900), reference)
+    expiry = from_iso(columns["expires_at"])
+    assert expiry > reference
+    assert (expiry - reference).days <= MAX_LIFETIME_DAYS
+
+
+def test_a_caller_cannot_extend_the_expiry_past_the_ceiling():
+    reference = now_utc()
+    far = reference.replace(year=reference.year + 5)
+    columns = task_columns(
+        Schedule(kind="interval", every_seconds=900, expires_at=far), reference
+    )
+    expiry = from_iso(columns["expires_at"])
+    assert (expiry - reference).days <= MAX_LIFETIME_DAYS

tests/unit/services/devii/tasks/scheduler.py

@@ -0,0 +1,194 @@
+# retoor <retoor@molodetz.nl>
+
+import asyncio
+
+from devplacepy.database import invalidate_admins_cache
+from devplacepy.services.devii.tasks.guards import (
+    REASON_EXPIRED,
+    REASON_MAX_RUNS,
+    REASON_NOT_ADMIN,
+)
+from devplacepy.services.devii.tasks.schedule import now_utc, to_iso
+from devplacepy.services.devii.tasks.scheduler import GlobalScheduler
+from devplacepy.services.devii.tasks.store import TaskStore
+from devplacepy.utils import generate_uid
+from tests.conftest import run_async
+
+
+def _account(local_db, role):
+    uid = generate_uid()
+    local_db["users"].insert(
+        {
+            "uid": uid,
+            "username": f"sched-{uid[-10:]}",
+            "role": role,
+            "api_key": "k",
+            "deleted_at": None,
+        }
+    )
+    invalidate_admins_cache()
+    return uid
+
+
+def _seed(local_db, owner, **overrides):
+    row = {
+        "uid": generate_uid(),
+        "owner_kind": "user",
+        "owner_id": owner,
+        "label": "scheduled",
+        "prompt": "work",
+        "enabled": True,
+        "status": "pending",
+        "kind": "interval",
+        "every_seconds": 900,
+        "max_runs": 10,
+        "run_count": 0,
+        "failure_count": 0,
+        "created_at": to_iso(now_utc()),
+        "next_run_at": to_iso(now_utc()),
+        "expires_at": None,
+        "deleted_at": None,
+        "deleted_by": None,
+    }
+    row.update(overrides)
+    local_db["devii_tasks"].insert(row)
+    return row["uid"]
+
+
+def _harness(local_db, started, gate, peak=None):
+    active: dict[str, int] = {}
+
+    def resolve(row):
+        owner = str(row["owner_id"])
+        store = TaskStore(local_db, "user", owner)
+
+        async def executor(prompt):
+            started.append(owner)
+            active[owner] = active.get(owner, 0) + 1
+            if peak is not None:
+                peak.append(max(active.values()))
+            try:
+                await gate.wait()
+                return f"done: {prompt}"
+            finally:
+                active[owner] -= 1
+
+        return store, executor, lambda kind, task_row, payload: None
+
+    return resolve
+
+
+def test_scheduler_runs_one_task_per_owner_and_completes_it(local_db):
+    local_db["devii_tasks"].delete()
+    owner = _account(local_db, "Admin")
+    first = _seed(local_db, owner)
+    second = _seed(local_db, owner)
+
+    started: list[str] = []
+    peak: list[int] = []
+
+    async def run():
+        gate = asyncio.Event()
+        scheduler = GlobalScheduler(
+            local_db, _harness(local_db, started, gate, peak), tick_seconds=0.05
+        )
+        scheduler.start()
+        await asyncio.sleep(0.4)
+        claimed = [r["uid"] for r in local_db["devii_tasks"].find(status="running")]
+        held = len(started)
+        gate.set()
+        await asyncio.sleep(0.5)
+        await scheduler.stop()
+        return claimed, held
+
+    claimed, held = run_async(run())
+    assert held == 1
+    assert max(peak) == 1
+    assert len(claimed) == 1
+    done = local_db["devii_tasks"].find_one(uid=claimed[0])
+    assert int(done["run_count"]) == 1
+    assert done["status"] == "pending"
+    assert done["last_result"].startswith("done:")
+    assert done["next_run_at"] > to_iso(now_utc())
+    other = second if claimed[0] == first else first
+    assert local_db["devii_tasks"].find_one(uid=other)["status"] == "pending"
+
+
+def test_scheduler_retires_a_task_whose_owner_may_not_schedule(local_db):
+    local_db["devii_tasks"].delete()
+    uid = _seed(local_db, _account(local_db, "Member"))
+    started: list[str] = []
+
+    async def run():
+        gate = asyncio.Event()
+        gate.set()
+        scheduler = GlobalScheduler(
+            local_db, _harness(local_db, started, gate), tick_seconds=0.05
+        )
+        scheduler.start()
+        await asyncio.sleep(0.3)
+        await scheduler.stop()
+
+    run_async(run())
+    row = local_db["devii_tasks"].find_one(uid=uid)
+    assert not row["enabled"]
+    assert row["status"] == "disabled"
+    assert row["last_error"] == REASON_NOT_ADMIN
+    assert started == []
+
+
+def test_scheduler_retires_expired_and_exhausted_tasks_even_while_saturated(local_db):
+    local_db["devii_tasks"].delete()
+    owner = _account(local_db, "Admin")
+    _seed(local_db, owner)
+    expired = _seed(
+        local_db, owner, expires_at=to_iso(now_utc().replace(year=now_utc().year - 1))
+    )
+    exhausted = _seed(local_db, owner, max_runs=3, run_count=3)
+    started: list[str] = []
+
+    async def run():
+        gate = asyncio.Event()
+        scheduler = GlobalScheduler(
+            local_db, _harness(local_db, started, gate), tick_seconds=0.05
+        )
+        scheduler.start()
+        await asyncio.sleep(0.4)
+        gate.set()
+        await asyncio.sleep(0.2)
+        await scheduler.stop()
+
+    run_async(run())
+    assert local_db["devii_tasks"].find_one(uid=expired)["last_error"] == REASON_EXPIRED
+    assert local_db["devii_tasks"].find_one(uid=exhausted)["last_error"] == REASON_MAX_RUNS
+    for uid in (expired, exhausted):
+        assert not local_db["devii_tasks"].find_one(uid=uid)["enabled"]
+
+
+def test_scheduler_disables_a_task_after_repeated_failures(local_db):
+    local_db["devii_tasks"].delete()
+    owner = _account(local_db, "Admin")
+    uid = _seed(local_db, owner, failure_count=2)
+
+    def resolve(row):
+        store = TaskStore(local_db, "user", str(row["owner_id"]))
+
+        async def executor(prompt):
+            raise RuntimeError("upstream is down")
+
+        return store, executor, lambda kind, task_row, payload: None
+
+    async def run():
+        scheduler = GlobalScheduler(
+            local_db, resolve, tick_seconds=0.05, max_failures=3
+        )
+        scheduler.start()
+        await asyncio.sleep(0.4)
+        await scheduler.stop()
+
+    run_async(run())
+    row = local_db["devii_tasks"].find_one(uid=uid)
+    assert int(row["failure_count"]) == 3
+    assert not row["enabled"]
+    assert row["status"] == "error"
+    assert "upstream is down" in row["last_error"]

tests/unit/services/devii/tasks/store.py

@@ -0,0 +1,136 @@
+# retoor <retoor@molodetz.nl>
+
+import pytest
+
+from devplacepy.database import invalidate_admins_cache
+from devplacepy.services.devii.errors import ToolInputError
+from devplacepy.services.devii.tasks.controller import TaskController
+from devplacepy.services.devii.tasks.guards import AutomationDenied
+from devplacepy.services.devii.tasks.schedule import now_utc, to_iso
+from devplacepy.services.devii.tasks.store import TaskStore, claim, due_rows
+from devplacepy.utils import generate_uid
+
+
+def _account(local_db, role):
+    uid = generate_uid()
+    local_db["users"].insert(
+        {
+            "uid": uid,
+            "username": f"store-{uid[-10:]}",
+            "role": role,
+            "deleted_at": None,
+        }
+    )
+    invalidate_admins_cache()
+    return uid
+
+
+def _record(**overrides):
+    record = {
+        "uid": generate_uid(),
+        "prompt": "work",
+        "enabled": True,
+        "status": "pending",
+        "created_at": to_iso(now_utc()),
+        "next_run_at": to_iso(now_utc()),
+        "run_count": 0,
+        "failure_count": 0,
+        "kind": "interval",
+        "every_seconds": 900,
+        "max_runs": 10,
+    }
+    record.update(overrides)
+    return record
+
+
+def test_member_cannot_persist_a_task(local_db):
+    store = TaskStore(local_db, "user", _account(local_db, "Member"))
+    with pytest.raises(AutomationDenied):
+        store.create(_record())
+
+
+def test_administrator_can_persist_a_task(local_db):
+    store = TaskStore(local_db, "user", _account(local_db, "Admin"))
+    record = _record()
+    store.create(record)
+    assert store.get(record["uid"]) is not None
+
+
+def test_member_cannot_re_enable_a_task(local_db):
+    admin = _account(local_db, "Admin")
+    member = _account(local_db, "Member")
+    admin_store = TaskStore(local_db, "user", admin)
+    record = _record()
+    admin_store.create(record)
+    local_db["devii_tasks"].update(
+        {"uid": record["uid"], "owner_id": member, "enabled": False}, ["uid"]
+    )
+    member_store = TaskStore(local_db, "user", member)
+    with pytest.raises(AutomationDenied):
+        member_store.update(record["uid"], {"enabled": True})
+    member_store.update(record["uid"], {"enabled": False, "status": "disabled"})
+
+
+def test_local_operator_store_bypasses_the_role_gate(local_db):
+    store = TaskStore(local_db, "user", "cli", operator=True)
+    record = _record()
+    store.create(record)
+    assert store.get(record["uid"]) is not None
+
+
+def test_claim_succeeds_once(local_db):
+    store = TaskStore(local_db, "user", _account(local_db, "Admin"))
+    record = _record()
+    store.create(record)
+    assert claim(local_db, record["uid"]) is True
+    assert claim(local_db, record["uid"]) is False
+
+
+def test_claim_refuses_a_disabled_task(local_db):
+    store = TaskStore(local_db, "user", _account(local_db, "Admin"))
+    record = _record()
+    store.create(record)
+    store.update(record["uid"], {"enabled": False, "status": "disabled"})
+    assert claim(local_db, record["uid"]) is False
+
+
+def test_due_rows_skips_future_and_running_tasks(local_db):
+    admin = _account(local_db, "Admin")
+    store = TaskStore(local_db, "user", admin)
+    now = now_utc()
+    due = _record(next_run_at=to_iso(now.replace(year=now.year - 1)))
+    future = _record(next_run_at=to_iso(now.replace(year=now.year + 1)))
+    running = _record(
+        status="running", next_run_at=to_iso(now.replace(year=now.year - 1))
+    )
+    for record in (due, future, running):
+        store.create(record)
+    found = {row["uid"] for row in due_rows(local_db, to_iso(now), 500)}
+    assert due["uid"] in found
+    assert future["uid"] not in found
+    assert running["uid"] not in found
+
+
+def test_controller_caps_active_tasks_per_owner(local_db, monkeypatch):
+    monkeypatch.setattr(
+        "devplacepy.services.devii.tasks.controller.max_active_per_owner", lambda: 2
+    )
+    store = TaskStore(local_db, "user", _account(local_db, "Admin"))
+    controller = TaskController(store)
+    for _ in range(2):
+        controller.create_task(
+            {"prompt": "work", "kind": "interval", "every_seconds": 900}
+        )
+    with pytest.raises(ToolInputError):
+        controller.create_task(
+            {"prompt": "work", "kind": "interval", "every_seconds": 900}
+        )
+
+
+def test_controller_refuses_a_member(local_db):
+    store = TaskStore(local_db, "user", _account(local_db, "Member"))
+    controller = TaskController(store)
+    with pytest.raises(ToolInputError):
+        controller.create_task(
+            {"prompt": "work", "kind": "interval", "every_seconds": 900}
+        )