This commit is contained in:
2026-07-26 17:24:49 +02:00
parent b8277d6351
commit a3963611f0
23 changed files with 1499 additions and 189 deletions
+25
View File
@@ -155,3 +155,28 @@ def test_member_cannot_disable_a_task(seeded_db):
)
assert response.status_code in (302, 303)
assert db["devii_tasks"].find_one(uid=uid)["enabled"]
def test_devii_tasks_reports_the_role_quotas(seeded_db):
admin = _admin_dt(seeded_db)
body = admin.get(f"{BASE_URL}/admin/devii-tasks", headers=JSON_dt).json()
assert body["limits"]["member_create_24h"] == 5
assert body["limits"]["member_runs_24h"] == 10
assert body["limits"]["admin_create_24h"] == 5
assert body["limits"]["admin_runs_24h"] == 100
def test_devii_tasks_shows_the_owner_24h_usage(seeded_db):
from devplacepy.database import db
from devplacepy.services.devii.tasks.limits import record_run
from devplacepy.services.devii.tasks.schedule import now_utc
owner = _admin_uid(seeded_db)
uid = _seed_task(owner)
record_run(db, "user", owner, uid, now_utc())
admin = _admin_dt(seeded_db)
body = admin.get(f"{BASE_URL}/admin/devii-tasks", headers=JSON_dt).json()
entry = next(item for item in body["items"] if item["uid"] == uid)
assert entry["quota"]["runs_used"] >= 1
assert entry["quota"]["runs_limit"] == 100
assert entry["quota"]["creates_used"] >= 1
+13 -10
View File
@@ -121,30 +121,33 @@ def test_game_read_actions_are_read_only():
assert BY_NAME[name].is_read_only is True
def test_scheduling_tools_are_administrator_only():
def test_scheduling_tools_need_an_account_but_not_an_administrator():
from devplacepy.services.devii.registry import CATALOG
by_name = CATALOG.by_name()
for name in ("create_task", "update_task", "run_task_now"):
action = by_name[name]
assert action.requires_admin is True
assert action.requires_auth is True
for name in ("list_tasks", "get_task", "delete_task"):
assert by_name[name].requires_admin is False
assert action.requires_admin is False
def test_scheduling_tools_hidden_from_a_member_tool_list():
def test_scheduling_tools_are_offered_to_a_member():
from devplacepy.services.devii.registry import CATALOG
member = {
s["function"]["name"]
for s in CATALOG.tool_schemas_for(authenticated=True, is_admin=False)
}
admin = {
for name in ("create_task", "update_task", "run_task_now", "list_tasks"):
assert name in member
def test_scheduling_tools_are_hidden_from_guests():
from devplacepy.services.devii.registry import CATALOG
guest = {
s["function"]["name"]
for s in CATALOG.tool_schemas_for(authenticated=True, is_admin=True)
for s in CATALOG.tool_schemas_for(authenticated=False)
}
for name in ("create_task", "update_task", "run_task_now"):
assert name not in member
assert name in admin
assert "list_tasks" in member
assert name not in guest
+55
View File
@@ -128,3 +128,58 @@ def test_parse_topic_extracts_embedded_json():
def test_parse_topic_defaults_to_followup_on_garbage():
assert _parse_topic("not json at all")[0] == "follow_up"
assert _parse_topic("")[0] == "follow_up"
def test_task_run_block_is_absent_outside_a_scheduled_run(local_db):
session = _session("main", "sess-taskblock-off")
assert "TASK RUN ENVIRONMENT" not in session._compose_system_prompt()
def test_member_session_is_told_it_may_not_nest_tasks(local_db):
from devplacepy.services.devii.tasks.context import task_run_scope
hub, svc = _hub()
local_db["users"].insert(
{
"uid": "sess-task-member",
"username": "task-member",
"role": "Member",
"api_key": "k",
"deleted_at": None,
"created_at": "2026-01-01T00:00:00",
}
)
database.invalidate_admins_cache()
session = hub.get_or_create(
"user", "sess-task-member", "task-member", "", svc.instance_base_url(),
channel="main",
)
with task_run_scope():
prompt = session._compose_system_prompt()
assert "TASK RUN ENVIRONMENT" in prompt
assert "may NOT schedule further work" in prompt
def test_administrator_session_is_told_it_may_nest_tasks(local_db):
from devplacepy.services.devii.tasks.context import task_run_scope
hub, svc = _hub()
local_db["users"].insert(
{
"uid": "sess-task-admin",
"username": "task-admin",
"role": "Admin",
"api_key": "k",
"deleted_at": None,
"created_at": "2026-01-01T00:00:00",
}
)
database.invalidate_admins_cache()
session = hub.get_or_create(
"user", "sess-task-admin", "task-admin", "", svc.instance_base_url(),
channel="main",
)
with task_run_scope():
prompt = session._compose_system_prompt()
assert "TASK RUN ENVIRONMENT" in prompt
assert "MAY" in prompt
+104
View File
@@ -0,0 +1,104 @@
# retoor <retoor@molodetz.nl>
import asyncio
from devplacepy.database import invalidate_admins_cache
from devplacepy.services.devii.tasks.context import in_task_run, task_run_scope
from devplacepy.services.devii.tasks.guards import nesting_allowed
from devplacepy.services.devii.tasks.schedule import now_utc, to_iso
from devplacepy.utils import generate_uid
from tests.conftest import run_async
def _account(local_db, role):
uid = generate_uid()
local_db["users"].insert(
{
"uid": uid,
"username": f"ctx-{uid[-10:]}",
"role": role,
"deleted_at": None,
"created_at": to_iso(now_utc()),
}
)
invalidate_admins_cache()
return uid
def test_outside_a_task_run_the_flag_is_off():
assert in_task_run() is False
def test_scope_sets_and_restores_the_flag():
assert in_task_run() is False
with task_run_scope():
assert in_task_run() is True
assert in_task_run() is False
def test_scope_restores_the_flag_after_an_exception():
try:
with task_run_scope():
raise RuntimeError("boom")
except RuntimeError:
pass
assert in_task_run() is False
def test_nested_scopes_restore_correctly():
with task_run_scope():
with task_run_scope():
assert in_task_run() is True
assert in_task_run() is True
assert in_task_run() is False
def test_the_flag_reaches_work_spawned_inside_the_run():
seen = []
async def inner():
seen.append(in_task_run())
async def run():
with task_run_scope():
await inner()
task = asyncio.create_task(inner())
await task
run_async(run())
assert seen == [True, True]
def test_a_concurrent_turn_outside_the_run_never_sees_the_flag():
seen = {}
async def scheduled(gate):
with task_run_scope():
seen["inside"] = in_task_run()
gate.set()
await asyncio.sleep(0.05)
seen["inside_after"] = in_task_run()
async def interactive(gate):
await gate.wait()
seen["outside"] = in_task_run()
async def run():
gate = asyncio.Event()
await asyncio.gather(scheduled(gate), interactive(gate))
run_async(run())
assert seen["inside"] is True
assert seen["inside_after"] is True
assert seen["outside"] is False
def test_members_may_not_nest_but_administrators_may(local_db):
member = _account(local_db, "Member")
admin = _account(local_db, "Admin")
assert nesting_allowed(member) is True
assert nesting_allowed(admin) is True
with task_run_scope():
assert nesting_allowed(member) is False
assert nesting_allowed(admin) is True
assert nesting_allowed(member) is True
+126 -34
View File
@@ -3,15 +3,24 @@
from datetime import timedelta
from devplacepy.database import invalidate_admins_cache
from devplacepy.services.devii.tasks import limits
from devplacepy.services.devii.tasks.context import task_run_scope
from devplacepy.services.devii.tasks.guards import (
REASON_BUDGET,
REASON_CREATE_QUOTA,
REASON_EXPIRED,
REASON_FAILURES,
REASON_MAX_RUNS,
REASON_NOT_ADMIN,
REASON_NESTED,
REASON_NOT_A_USER,
REASON_RUN_QUOTA,
automation_allowed,
refusal,
budget_deferral,
creation_denial,
retire_reason,
run_deferral,
)
from devplacepy.services.devii.tasks.limits import reserve_run
from devplacepy.services.devii.tasks.schedule import now_utc, to_iso
from devplacepy.utils import generate_uid
@@ -24,6 +33,7 @@ def _account(local_db, role):
"username": f"guard-{uid[-10:]}",
"role": role,
"deleted_at": None,
"created_at": to_iso(now_utc()),
}
)
invalidate_admins_cache()
@@ -42,54 +52,136 @@ def _row(owner_uid, **overrides):
return row
def test_only_administrators_may_automate(local_db):
admin = _account(local_db, "Admin")
def _burn_runs(local_db, owner, count, reference):
for index in range(count):
limits.record_run(
local_db,
"user",
owner,
generate_uid(),
reference - timedelta(minutes=index),
)
def test_any_signed_in_account_may_automate(local_db):
member = _account(local_db, "Member")
admin = _account(local_db, "Admin")
assert automation_allowed("user", member) is True
assert automation_allowed("user", admin) is True
assert automation_allowed("user", member) is False
assert automation_allowed("guest", "guest-cookie") is False
assert automation_allowed("user", "") is False
def test_member_owned_task_is_refused(local_db):
def test_member_task_is_no_longer_retired_for_its_role(local_db):
member = _account(local_db, "Member")
assert refusal(_row(member), now_utc()) == REASON_NOT_ADMIN
assert retire_reason(_row(member), now_utc()) is None
def test_clean_admin_task_is_allowed(local_db):
admin = _account(local_db, "Admin")
assert refusal(_row(admin), now_utc()) is None
def test_guest_owned_task_is_retired(local_db):
row = _row("cookie", owner_kind="guest")
assert retire_reason(row, now_utc()) == REASON_NOT_A_USER
def test_exhausted_task_is_refused(local_db):
admin = _account(local_db, "Admin")
row = _row(admin, max_runs=5, run_count=5)
assert refusal(row, now_utc()) == REASON_MAX_RUNS
def test_expired_task_is_refused(local_db):
admin = _account(local_db, "Admin")
def test_expired_task_is_retired(local_db):
owner = _account(local_db, "Member")
now = now_utc()
row = _row(admin, expires_at=to_iso(now - timedelta(minutes=1)))
assert refusal(row, now) == REASON_EXPIRED
row = _row(owner, expires_at=to_iso(now - timedelta(minutes=1)))
assert retire_reason(row, now) == REASON_EXPIRED
def test_task_without_an_expiry_stops_at_the_fallback_ceiling(local_db):
admin = _account(local_db, "Admin")
def test_exhausted_task_is_retired(local_db):
owner = _account(local_db, "Member")
assert retire_reason(_row(owner, max_runs=5, run_count=5), now_utc()) == REASON_MAX_RUNS
def test_repeated_failures_retire_the_task(local_db):
owner = _account(local_db, "Member")
row = _row(owner, failure_count=3)
assert retire_reason(row, now_utc(), max_failures=3) == REASON_FAILURES
assert retire_reason(row, now_utc(), max_failures=0) is None
def test_reservation_succeeds_until_the_quota_is_spent(local_db):
owner = _account(local_db, "Member")
now = now_utc()
row = _row(admin, created_at=to_iso(now - timedelta(days=40)))
assert refusal(row, now) == REASON_EXPIRED
granted = [
reserve_run(local_db, "user", owner, generate_uid(), now) for _ in range(12)
]
assert granted.count(True) == 10
assert granted.count(False) == 2
def test_repeated_failures_refuse_the_task(local_db):
admin = _account(local_db, "Admin")
row = _row(admin, failure_count=3)
assert refusal(row, now_utc(), max_failures=3) == REASON_FAILURES
assert refusal(row, now_utc(), max_failures=0) is None
def test_a_spent_quota_defers_instead_of_retiring(local_db):
owner = _account(local_db, "Member")
now = now_utc()
row = _row(owner)
_burn_runs(local_db, owner, 10, now)
assert reserve_run(local_db, "user", owner, row["uid"], now) is False
postponement = run_deferral(local_db, row, now)
assert postponement.reason == REASON_RUN_QUOTA
assert postponement.retry_at > now
assert retire_reason(row, now) is None
def test_budget_probe_refuses_the_task(local_db):
admin = _account(local_db, "Admin")
row = _row(admin)
assert refusal(row, now_utc(), budget_exceeded=lambda k, i: True) == REASON_BUDGET
assert refusal(row, now_utc(), budget_exceeded=lambda k, i: False) is None
def test_administrator_run_quota_is_larger(local_db):
owner = _account(local_db, "Admin")
now = now_utc()
_burn_runs(local_db, owner, 10, now)
assert reserve_run(local_db, "user", owner, generate_uid(), now) is True
def test_budget_defers_the_task(local_db):
owner = _account(local_db, "Member")
now = now_utc()
postponement = budget_deferral(
_row(owner), now, budget_exceeded=lambda kind, uid: True
)
assert postponement is not None
assert postponement.reason == REASON_BUDGET
assert postponement.retry_at > now
assert budget_deferral(_row(owner), now, lambda kind, uid: False) is None
assert budget_deferral(_row(owner), now, None) is None
def test_creation_is_allowed_within_the_quota(local_db):
owner = _account(local_db, "Member")
assert creation_denial(local_db, "user", owner, now_utc()) is None
def test_creation_is_denied_over_the_quota(local_db):
owner = _account(local_db, "Member")
now = now_utc()
for _ in range(5):
local_db["devii_tasks"].insert(
{
"uid": generate_uid(),
"owner_kind": "user",
"owner_id": owner,
"created_at": to_iso(now),
"deleted_at": None,
}
)
denial = creation_denial(local_db, "user", owner, now)
assert denial is not None
assert denial.reason == REASON_CREATE_QUOTA
assert denial.retry_at is not None
def test_creation_inside_a_task_run_is_denied_for_a_member(local_db):
owner = _account(local_db, "Member")
with task_run_scope():
denial = creation_denial(local_db, "user", owner, now_utc())
assert denial is not None
assert denial.reason == REASON_NESTED
def test_creation_inside_a_task_run_is_allowed_for_an_administrator(local_db):
owner = _account(local_db, "Admin")
with task_run_scope():
assert creation_denial(local_db, "user", owner, now_utc()) is None
def test_guest_creation_is_denied(local_db):
denial = creation_denial(local_db, "guest", "cookie", now_utc())
assert denial is not None
assert denial.reason == REASON_NOT_A_USER
+196
View File
@@ -0,0 +1,196 @@
# retoor <retoor@molodetz.nl>
from datetime import timedelta
from devplacepy.database import invalidate_admins_cache, set_setting
from devplacepy.services.devii.tasks import limits
from devplacepy.services.devii.tasks.schedule import now_utc, to_iso
from devplacepy.utils import generate_uid
def _account(local_db, role):
uid = generate_uid()
local_db["users"].insert(
{
"uid": uid,
"username": f"lim-{uid[-10:]}",
"role": role,
"deleted_at": None,
"created_at": to_iso(now_utc()),
}
)
invalidate_admins_cache()
return uid
def _run_at(local_db, owner, moment):
limits.record_run(local_db, "user", owner, generate_uid(), moment)
def _task_at(local_db, owner, moment, deleted=None):
local_db["devii_tasks"].insert(
{
"uid": generate_uid(),
"owner_kind": "user",
"owner_id": owner,
"prompt": "work",
"enabled": True,
"status": "pending",
"created_at": to_iso(moment),
"deleted_at": deleted,
}
)
def test_default_limits_follow_the_role(local_db):
assert limits.create_limit(False) == 5
assert limits.run_limit(False) == 10
assert limits.create_limit(True) == 5
assert limits.run_limit(True) == 100
def test_limits_are_configurable(local_db):
set_setting(limits.FIELD_MEMBER_RUNS, "3")
try:
assert limits.run_limit(False) == 3
finally:
set_setting(limits.FIELD_MEMBER_RUNS, str(limits.DEFAULT_MEMBER_RUNS))
assert limits.run_limit(False) == limits.DEFAULT_MEMBER_RUNS
def test_run_quota_counts_only_the_window(local_db):
owner = _account(local_db, "Member")
now = now_utc()
for offset in (1, 2, 3):
_run_at(local_db, owner, now - timedelta(hours=offset))
_run_at(local_db, owner, now - timedelta(hours=30))
quota = limits.run_quota(local_db, "user", owner, now)
assert quota.used == 3
assert quota.limit == 10
assert quota.exceeded is False
assert quota.remaining == 7
def test_run_quota_reports_when_the_next_slot_frees(local_db):
owner = _account(local_db, "Member")
now = now_utc()
oldest = now - timedelta(hours=20)
_run_at(local_db, owner, oldest)
for index in range(9):
_run_at(local_db, owner, now - timedelta(hours=1, minutes=index))
quota = limits.run_quota(local_db, "user", owner, now)
assert quota.used == 10
assert quota.exceeded is True
assert quota.remaining == 0
assert quota.free_at == oldest + timedelta(hours=24)
def test_run_quota_is_per_owner(local_db):
first = _account(local_db, "Member")
second = _account(local_db, "Member")
now = now_utc()
for _ in range(10):
_run_at(local_db, first, now)
assert limits.run_quota(local_db, "user", first, now).exceeded is True
assert limits.run_quota(local_db, "user", second, now).used == 0
def test_administrators_get_the_larger_run_quota(local_db):
owner = _account(local_db, "Admin")
now = now_utc()
for _ in range(50):
_run_at(local_db, owner, now)
quota = limits.run_quota(local_db, "user", owner, now)
assert quota.limit == 100
assert quota.exceeded is False
def test_create_quota_counts_tasks_in_the_window(local_db):
owner = _account(local_db, "Member")
now = now_utc()
for offset in range(4):
_task_at(local_db, owner, now - timedelta(hours=offset))
_task_at(local_db, owner, now - timedelta(hours=25))
quota = limits.create_quota(local_db, "user", owner, now)
assert quota.used == 4
assert quota.limit == 5
assert quota.exceeded is False
def test_create_quota_still_counts_a_deleted_task(local_db):
owner = _account(local_db, "Member")
now = now_utc()
for _ in range(5):
_task_at(local_db, owner, now, deleted=to_iso(now))
quota = limits.create_quota(local_db, "user", owner, now)
assert quota.used == 5
assert quota.exceeded is True
def test_zero_limit_means_unlimited(local_db):
owner = _account(local_db, "Member")
now = now_utc()
set_setting(limits.FIELD_MEMBER_RUNS, "0")
try:
for _ in range(30):
_run_at(local_db, owner, now)
quota = limits.run_quota(local_db, "user", owner, now)
assert quota.limit == 0
assert quota.exceeded is False
assert quota.remaining == -1
finally:
set_setting(limits.FIELD_MEMBER_RUNS, str(limits.DEFAULT_MEMBER_RUNS))
def test_reserve_run_grants_exactly_the_quota(local_db):
owner = _account(local_db, "Member")
now = now_utc()
granted = [
limits.reserve_run(local_db, "user", owner, generate_uid(), now)
for _ in range(15)
]
assert granted.count(True) == limits.DEFAULT_MEMBER_RUNS
assert granted[limits.DEFAULT_MEMBER_RUNS] is False
assert limits.run_quota(local_db, "user", owner, now).used == limits.DEFAULT_MEMBER_RUNS
def test_reserve_run_grants_an_administrator_more(local_db):
owner = _account(local_db, "Admin")
now = now_utc()
granted = [
limits.reserve_run(local_db, "user", owner, generate_uid(), now)
for _ in range(limits.DEFAULT_MEMBER_RUNS + 5)
]
assert all(granted)
def test_reserve_run_is_unbounded_when_the_limit_is_zero(local_db):
owner = _account(local_db, "Member")
now = now_utc()
set_setting(limits.FIELD_MEMBER_RUNS, "0")
try:
granted = [
limits.reserve_run(local_db, "user", owner, generate_uid(), now)
for _ in range(25)
]
assert all(granted)
finally:
set_setting(limits.FIELD_MEMBER_RUNS, str(limits.DEFAULT_MEMBER_RUNS))
def test_reserve_run_ignores_rows_outside_the_window(local_db):
owner = _account(local_db, "Member")
now = now_utc()
for index in range(limits.DEFAULT_MEMBER_RUNS):
_run_at(local_db, owner, now - timedelta(hours=25, minutes=index))
assert limits.reserve_run(local_db, "user", owner, generate_uid(), now) is True
def test_prune_drops_rows_outside_the_retention_window(local_db):
owner = _account(local_db, "Member")
now = now_utc()
_run_at(local_db, owner, now)
_run_at(local_db, owner, now - timedelta(hours=72))
removed = limits.prune_runs(local_db, now)
assert removed >= 1
assert limits.run_quota(local_db, "user", owner, now).used == 1
+177 -46
View File
@@ -1,12 +1,17 @@
# retoor <retoor@molodetz.nl>
import asyncio
from datetime import timedelta
from devplacepy.database import invalidate_admins_cache
from devplacepy.services.devii.tasks import limits
from devplacepy.services.devii.tasks.context import in_task_run
from devplacepy.services.devii.tasks.guards import (
REASON_BUDGET,
REASON_EXPIRED,
REASON_MAX_RUNS,
REASON_NOT_ADMIN,
REASON_NOT_A_USER,
REASON_RUN_QUOTA,
)
from devplacepy.services.devii.tasks.schedule import now_utc, to_iso
from devplacepy.services.devii.tasks.scheduler import GlobalScheduler
@@ -24,16 +29,17 @@ def _account(local_db, role):
"role": role,
"api_key": "k",
"deleted_at": None,
"created_at": to_iso(now_utc()),
}
)
invalidate_admins_cache()
return uid
def _seed(local_db, owner, **overrides):
def _seed(local_db, owner, owner_kind="user", **overrides):
row = {
"uid": generate_uid(),
"owner_kind": "user",
"owner_kind": owner_kind,
"owner_id": owner,
"label": "scheduled",
"prompt": "work",
@@ -55,7 +61,7 @@ def _seed(local_db, owner, **overrides):
return row["uid"]
def _harness(local_db, started, gate, peak=None):
def _harness(local_db, started, gate, peak=None, flags=None):
active: dict[str, int] = {}
def resolve(row):
@@ -64,6 +70,8 @@ def _harness(local_db, started, gate, peak=None):
async def executor(prompt):
started.append(owner)
if flags is not None:
flags.append(in_task_run())
active[owner] = active.get(owner, 0) + 1
if peak is not None:
peak.append(max(active.values()))
@@ -78,6 +86,22 @@ def _harness(local_db, started, gate, peak=None):
return resolve
def _run_scheduler(local_db, resolve, hold=0.4, release=0.5, **kwargs):
async def run():
gate = kwargs.pop("gate", None) or asyncio.Event()
scheduler = GlobalScheduler(local_db, resolve, tick_seconds=0.05, **kwargs)
scheduler.start()
await asyncio.sleep(hold)
claimed = [r["uid"] for r in local_db["devii_tasks"].find(status="running")]
held = len(claimed)
gate.set()
await asyncio.sleep(release)
await scheduler.stop()
return claimed, held
return run
def test_scheduler_runs_one_task_per_owner_and_completes_it(local_db):
local_db["devii_tasks"].delete()
owner = _account(local_db, "Admin")
@@ -86,25 +110,15 @@ def test_scheduler_runs_one_task_per_owner_and_completes_it(local_db):
started: list[str] = []
peak: list[int] = []
gate = asyncio.Event()
async def run():
gate = asyncio.Event()
scheduler = GlobalScheduler(
local_db, _harness(local_db, started, gate, peak), tick_seconds=0.05
)
scheduler.start()
await asyncio.sleep(0.4)
claimed = [r["uid"] for r in local_db["devii_tasks"].find(status="running")]
held = len(started)
gate.set()
await asyncio.sleep(0.5)
await scheduler.stop()
return claimed, held
claimed, held = run_async(run())
claimed, held = run_async(
_run_scheduler(
local_db, _harness(local_db, started, gate, peak), gate=gate
)()
)
assert held == 1
assert max(peak) == 1
assert len(claimed) == 1
done = local_db["devii_tasks"].find_one(uid=claimed[0])
assert int(done["run_count"]) == 1
assert done["status"] == "pending"
@@ -114,31 +128,131 @@ def test_scheduler_runs_one_task_per_owner_and_completes_it(local_db):
assert local_db["devii_tasks"].find_one(uid=other)["status"] == "pending"
def test_scheduler_retires_a_task_whose_owner_may_not_schedule(local_db):
def test_scheduler_runs_a_member_task(local_db):
local_db["devii_tasks"].delete()
uid = _seed(local_db, _account(local_db, "Member"))
owner = _account(local_db, "Member")
uid = _seed(local_db, owner)
started: list[str] = []
gate = asyncio.Event()
async def run():
gate = asyncio.Event()
gate.set()
scheduler = GlobalScheduler(
local_db, _harness(local_db, started, gate), tick_seconds=0.05
run_async(_run_scheduler(local_db, _harness(local_db, started, gate), gate=gate)())
assert started == [owner]
row = local_db["devii_tasks"].find_one(uid=uid)
assert row["enabled"]
assert int(row["run_count"]) == 1
def test_the_executor_runs_inside_the_task_run_context(local_db):
local_db["devii_tasks"].delete()
owner = _account(local_db, "Member")
_seed(local_db, owner)
started: list[str] = []
flags: list[bool] = []
gate = asyncio.Event()
run_async(
_run_scheduler(
local_db, _harness(local_db, started, gate, flags=flags), gate=gate
)()
)
assert flags == [True]
assert in_task_run() is False
def test_every_run_is_recorded_against_the_owner_quota(local_db):
local_db["devii_tasks"].delete()
local_db[limits.RUNS_TABLE].delete()
owner = _account(local_db, "Member")
_seed(local_db, owner)
started: list[str] = []
gate = asyncio.Event()
run_async(_run_scheduler(local_db, _harness(local_db, started, gate), gate=gate)())
quota = limits.run_quota(local_db, "user", owner, now_utc())
assert quota.used == 1
assert quota.limit == limits.DEFAULT_MEMBER_RUNS
def test_a_member_over_the_run_quota_is_postponed_not_disabled(local_db):
local_db["devii_tasks"].delete()
local_db[limits.RUNS_TABLE].delete()
owner = _account(local_db, "Member")
uid = _seed(local_db, owner)
now = now_utc()
for index in range(limits.DEFAULT_MEMBER_RUNS):
limits.record_run(
local_db, "user", owner, generate_uid(), now - timedelta(minutes=index)
)
scheduler.start()
await asyncio.sleep(0.3)
await scheduler.stop()
started: list[str] = []
gate = asyncio.Event()
run_async(run())
run_async(
_run_scheduler(local_db, _harness(local_db, started, gate), gate=gate)()
)
row = local_db["devii_tasks"].find_one(uid=uid)
assert started == []
assert row["enabled"]
assert row["status"] == "pending"
assert row["last_error"] == REASON_RUN_QUOTA
assert row["next_run_at"] > to_iso(now)
def test_an_administrator_keeps_running_past_the_member_limit(local_db):
local_db["devii_tasks"].delete()
local_db[limits.RUNS_TABLE].delete()
owner = _account(local_db, "Admin")
_seed(local_db, owner)
now = now_utc()
for index in range(limits.DEFAULT_MEMBER_RUNS + 5):
limits.record_run(
local_db, "user", owner, generate_uid(), now - timedelta(minutes=index)
)
started: list[str] = []
gate = asyncio.Event()
run_async(_run_scheduler(local_db, _harness(local_db, started, gate), gate=gate)())
assert started == [owner]
def test_budget_postpones_the_task(local_db):
local_db["devii_tasks"].delete()
local_db[limits.RUNS_TABLE].delete()
owner = _account(local_db, "Admin")
uid = _seed(local_db, owner)
started: list[str] = []
gate = asyncio.Event()
run_async(
_run_scheduler(
local_db,
_harness(local_db, started, gate),
gate=gate,
budget_exceeded=lambda kind, owner_id: True,
)()
)
row = local_db["devii_tasks"].find_one(uid=uid)
assert started == []
assert row["enabled"]
assert row["last_error"] == REASON_BUDGET
def test_scheduler_retires_a_guest_owned_task(local_db):
local_db["devii_tasks"].delete()
uid = _seed(local_db, "guest-cookie", owner_kind="guest")
started: list[str] = []
gate = asyncio.Event()
gate.set()
run_async(_run_scheduler(local_db, _harness(local_db, started, gate), gate=gate)())
row = local_db["devii_tasks"].find_one(uid=uid)
assert not row["enabled"]
assert row["status"] == "disabled"
assert row["last_error"] == REASON_NOT_ADMIN
assert row["last_error"] == REASON_NOT_A_USER
assert started == []
def test_scheduler_retires_expired_and_exhausted_tasks_even_while_saturated(local_db):
local_db["devii_tasks"].delete()
local_db[limits.RUNS_TABLE].delete()
owner = _account(local_db, "Admin")
_seed(local_db, owner)
expired = _seed(
@@ -146,27 +260,20 @@ def test_scheduler_retires_expired_and_exhausted_tasks_even_while_saturated(loca
)
exhausted = _seed(local_db, owner, max_runs=3, run_count=3)
started: list[str] = []
gate = asyncio.Event()
async def run():
gate = asyncio.Event()
scheduler = GlobalScheduler(
local_db, _harness(local_db, started, gate), tick_seconds=0.05
)
scheduler.start()
await asyncio.sleep(0.4)
gate.set()
await asyncio.sleep(0.2)
await scheduler.stop()
run_async(run())
run_async(_run_scheduler(local_db, _harness(local_db, started, gate), gate=gate)())
assert local_db["devii_tasks"].find_one(uid=expired)["last_error"] == REASON_EXPIRED
assert local_db["devii_tasks"].find_one(uid=exhausted)["last_error"] == REASON_MAX_RUNS
assert (
local_db["devii_tasks"].find_one(uid=exhausted)["last_error"] == REASON_MAX_RUNS
)
for uid in (expired, exhausted):
assert not local_db["devii_tasks"].find_one(uid=uid)["enabled"]
def test_scheduler_disables_a_task_after_repeated_failures(local_db):
local_db["devii_tasks"].delete()
local_db[limits.RUNS_TABLE].delete()
owner = _account(local_db, "Admin")
uid = _seed(local_db, owner, failure_count=2)
@@ -192,3 +299,27 @@ def test_scheduler_disables_a_task_after_repeated_failures(local_db):
assert not row["enabled"]
assert row["status"] == "error"
assert "upstream is down" in row["last_error"]
def test_a_failed_run_still_counts_against_the_quota(local_db):
local_db["devii_tasks"].delete()
local_db[limits.RUNS_TABLE].delete()
owner = _account(local_db, "Member")
_seed(local_db, owner)
def resolve(row):
store = TaskStore(local_db, "user", str(row["owner_id"]))
async def executor(prompt):
raise RuntimeError("nope")
return store, executor, lambda kind, task_row, payload: None
async def run():
scheduler = GlobalScheduler(local_db, resolve, tick_seconds=0.05)
scheduler.start()
await asyncio.sleep(0.3)
await scheduler.stop()
run_async(run())
assert limits.run_quota(local_db, "user", owner, now_utc()).used >= 1
+143 -27
View File
@@ -2,10 +2,17 @@
import pytest
from devplacepy.database import invalidate_admins_cache
from devplacepy.database import invalidate_admins_cache, set_setting
from devplacepy.services.devii.errors import ToolInputError
from devplacepy.services.devii.tasks import limits
from devplacepy.services.devii.tasks.context import task_run_scope
from devplacepy.services.devii.tasks.controller import TaskController
from devplacepy.services.devii.tasks.guards import AutomationDenied
from devplacepy.services.devii.tasks.guards import (
REASON_CREATE_QUOTA,
REASON_NESTED,
REASON_NOT_A_USER,
AutomationDenied,
)
from devplacepy.services.devii.tasks.schedule import now_utc, to_iso
from devplacepy.services.devii.tasks.store import TaskStore, claim, due_rows
from devplacepy.utils import generate_uid
@@ -19,6 +26,7 @@ def _account(local_db, role):
"username": f"store-{uid[-10:]}",
"role": role,
"deleted_at": None,
"created_at": to_iso(now_utc()),
}
)
invalidate_admins_cache()
@@ -43,39 +51,116 @@ def _record(**overrides):
return record
def test_member_cannot_persist_a_task(local_db):
def _spend_creation_quota(store, count):
for _ in range(count):
store.create(_record())
def test_member_can_create_a_task(local_db):
store = TaskStore(local_db, "user", _account(local_db, "Member"))
record = _record()
store.create(record)
assert store.get(record["uid"]) is not None
def test_member_creation_stops_at_the_daily_limit(local_db):
store = TaskStore(local_db, "user", _account(local_db, "Member"))
_spend_creation_quota(store, limits.DEFAULT_MEMBER_CREATE)
with pytest.raises(AutomationDenied) as denied:
store.create(_record())
assert denied.value.reason == REASON_CREATE_QUOTA
assert denied.value.retry_at is not None
def test_deleting_a_task_does_not_give_the_creation_slot_back(local_db):
store = TaskStore(local_db, "user", _account(local_db, "Member"))
records = [_record() for _ in range(limits.DEFAULT_MEMBER_CREATE)]
for record in records:
store.create(record)
for record in records:
store.delete(record["uid"])
with pytest.raises(AutomationDenied):
store.create(_record())
def test_administrator_can_persist_a_task(local_db):
def test_administrator_creation_stops_at_its_own_limit(local_db):
store = TaskStore(local_db, "user", _account(local_db, "Admin"))
_spend_creation_quota(store, limits.DEFAULT_ADMIN_CREATE)
with pytest.raises(AutomationDenied) as denied:
store.create(_record())
assert denied.value.reason == REASON_CREATE_QUOTA
def test_creation_limit_is_configurable(local_db):
store = TaskStore(local_db, "user", _account(local_db, "Member"))
set_setting(limits.FIELD_MEMBER_CREATE, "1")
try:
store.create(_record())
with pytest.raises(AutomationDenied):
store.create(_record())
finally:
set_setting(limits.FIELD_MEMBER_CREATE, str(limits.DEFAULT_MEMBER_CREATE))
def test_guest_cannot_persist_a_task(local_db):
store = TaskStore(local_db, "guest", "guest-cookie")
with pytest.raises(AutomationDenied) as denied:
store.create(_record())
assert denied.value.reason == REASON_NOT_A_USER
def test_member_cannot_create_a_task_from_inside_a_task_run(local_db):
store = TaskStore(local_db, "user", _account(local_db, "Member"))
with task_run_scope():
with pytest.raises(AutomationDenied) as denied:
store.create(_record())
assert denied.value.reason == REASON_NESTED
def test_administrator_can_create_a_task_from_inside_a_task_run(local_db):
store = TaskStore(local_db, "user", _account(local_db, "Admin"))
record = _record()
with task_run_scope():
store.create(record)
assert store.get(record["uid"]) is not None
def test_member_cannot_enable_a_task_from_inside_a_task_run(local_db):
store = TaskStore(local_db, "user", _account(local_db, "Member"))
record = _record()
store.create(record)
store.update(record["uid"], {"enabled": False, "status": "disabled"})
with task_run_scope():
with pytest.raises(AutomationDenied) as denied:
store.update(record["uid"], {"enabled": True})
assert denied.value.reason == REASON_NESTED
def test_administrator_can_enable_a_task_from_inside_a_task_run(local_db):
store = TaskStore(local_db, "user", _account(local_db, "Admin"))
record = _record()
store.create(record)
assert store.get(record["uid"]) is not None
store.update(record["uid"], {"enabled": False, "status": "disabled"})
with task_run_scope():
store.update(record["uid"], {"enabled": True})
assert store.get(record["uid"])["enabled"]
def test_member_cannot_re_enable_a_task(local_db):
admin = _account(local_db, "Admin")
member = _account(local_db, "Member")
admin_store = TaskStore(local_db, "user", admin)
record = _record()
admin_store.create(record)
local_db["devii_tasks"].update(
{"uid": record["uid"], "owner_id": member, "enabled": False}, ["uid"]
)
member_store = TaskStore(local_db, "user", member)
with pytest.raises(AutomationDenied):
member_store.update(record["uid"], {"enabled": True})
member_store.update(record["uid"], {"enabled": False, "status": "disabled"})
def test_local_operator_store_bypasses_the_role_gate(local_db):
store = TaskStore(local_db, "user", "cli", operator=True)
def test_disabling_from_inside_a_task_run_is_always_allowed(local_db):
store = TaskStore(local_db, "user", _account(local_db, "Member"))
record = _record()
store.create(record)
assert store.get(record["uid"]) is not None
with task_run_scope():
store.update(record["uid"], {"enabled": False, "status": "disabled"})
assert not store.get(record["uid"])["enabled"]
def test_local_operator_store_bypasses_every_gate(local_db):
store = TaskStore(local_db, "user", "cli", operator=True)
with task_run_scope():
for _ in range(limits.DEFAULT_MEMBER_CREATE + 3):
store.create(_record())
assert store.count_active() >= limits.DEFAULT_MEMBER_CREATE
def test_claim_succeeds_once(local_db):
@@ -95,8 +180,7 @@ def test_claim_refuses_a_disabled_task(local_db):
def test_due_rows_skips_future_and_running_tasks(local_db):
admin = _account(local_db, "Admin")
store = TaskStore(local_db, "user", admin)
store = TaskStore(local_db, "user", _account(local_db, "Admin"))
now = now_utc()
due = _record(next_run_at=to_iso(now.replace(year=now.year - 1)))
future = _record(next_run_at=to_iso(now.replace(year=now.year + 1)))
@@ -127,10 +211,42 @@ def test_controller_caps_active_tasks_per_owner(local_db, monkeypatch):
)
def test_controller_refuses_a_member(local_db):
def test_controller_reports_when_the_next_creation_slot_frees(local_db):
store = TaskStore(local_db, "user", _account(local_db, "Member"))
controller = TaskController(store)
with pytest.raises(ToolInputError):
for _ in range(limits.DEFAULT_MEMBER_CREATE):
controller.create_task(
{"prompt": "work", "kind": "interval", "every_seconds": 900}
)
with pytest.raises(ToolInputError) as failure:
controller.create_task(
{"prompt": "work", "kind": "interval", "every_seconds": 900}
)
assert REASON_CREATE_QUOTA in str(failure.value)
assert "frees up at" in str(failure.value)
def test_controller_refuses_nested_creation_for_a_member(local_db):
store = TaskStore(local_db, "user", _account(local_db, "Member"))
controller = TaskController(store)
with task_run_scope():
with pytest.raises(ToolInputError) as failure:
controller.create_task(
{"prompt": "work", "kind": "interval", "every_seconds": 900}
)
assert REASON_NESTED in str(failure.value)
def test_controller_refuses_nested_run_now_for_a_member(local_db):
store = TaskStore(local_db, "user", _account(local_db, "Member"))
controller = TaskController(store)
created = controller.create_task(
{"prompt": "work", "kind": "interval", "every_seconds": 900}
)
import json
uid = json.loads(created)["task"]["uid"]
with task_run_scope():
with pytest.raises(ToolInputError) as failure:
controller.run_task_now({"uid": uid})
assert REASON_NESTED in str(failure.value)