retoor/devplacepy
1
0
Fork 1
Code Issues 44 Pull Requests 4 Actions Packages Projects Releases Wiki Activity

ipdate

Browse Source
This commit is contained in:
retoor
2026-07-26 17:24:49 +02:00
parent b8277d6351
commit a3963611f0
23 changed files with 1499 additions and 189 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/unit/services/devii/tasks/scheduler.py
+177 -46
View File
@@ -1,12 +1,17 @@
# retoor <retoor@molodetz.nl>
import asyncio
from datetime import timedelta
from devplacepy.database import invalidate_admins_cache
from devplacepy.services.devii.tasks import limits
from devplacepy.services.devii.tasks.context import in_task_run
from devplacepy.services.devii.tasks.guards import (
REASON_BUDGET,
REASON_EXPIRED,
REASON_MAX_RUNS,
REASON_NOT_ADMIN,
REASON_NOT_A_USER,
REASON_RUN_QUOTA,
)
from devplacepy.services.devii.tasks.schedule import now_utc, to_iso
from devplacepy.services.devii.tasks.scheduler import GlobalScheduler
@@ -24,16 +29,17 @@ def _account(local_db, role):
"role": role,
"api_key": "k",
"deleted_at": None,
"created_at": to_iso(now_utc()),
}
)
invalidate_admins_cache()
return uid
def _seed(local_db, owner, **overrides):
def _seed(local_db, owner, owner_kind="user", **overrides):
row = {
"uid": generate_uid(),
"owner_kind": "user",
"owner_kind": owner_kind,
"owner_id": owner,
"label": "scheduled",
"prompt": "work",
@@ -55,7 +61,7 @@ def _seed(local_db, owner, **overrides):
return row["uid"]
def _harness(local_db, started, gate, peak=None):
def _harness(local_db, started, gate, peak=None, flags=None):
active: dict[str, int] = {}
def resolve(row):
@@ -64,6 +70,8 @@ def _harness(local_db, started, gate, peak=None):
async def executor(prompt):
started.append(owner)
if flags is not None:
flags.append(in_task_run())
active[owner] = active.get(owner, 0) + 1
if peak is not None:
peak.append(max(active.values()))
@@ -78,6 +86,22 @@ def _harness(local_db, started, gate, peak=None):
return resolve
def _run_scheduler(local_db, resolve, hold=0.4, release=0.5, **kwargs):
async def run():
gate = kwargs.pop("gate", None) or asyncio.Event()
scheduler = GlobalScheduler(local_db, resolve, tick_seconds=0.05, **kwargs)
scheduler.start()
await asyncio.sleep(hold)
claimed = [r["uid"] for r in local_db["devii_tasks"].find(status="running")]
held = len(claimed)
gate.set()
await asyncio.sleep(release)
await scheduler.stop()
return claimed, held
return run
def test_scheduler_runs_one_task_per_owner_and_completes_it(local_db):
local_db["devii_tasks"].delete()
owner = _account(local_db, "Admin")
@@ -86,25 +110,15 @@ def test_scheduler_runs_one_task_per_owner_and_completes_it(local_db):
started: list[str] = []
peak: list[int] = []
gate = asyncio.Event()
async def run():
gate = asyncio.Event()
scheduler = GlobalScheduler(
local_db, _harness(local_db, started, gate, peak), tick_seconds=0.05
)
scheduler.start()
await asyncio.sleep(0.4)
claimed = [r["uid"] for r in local_db["devii_tasks"].find(status="running")]
held = len(started)
gate.set()
await asyncio.sleep(0.5)
await scheduler.stop()
return claimed, held
claimed, held = run_async(run())
claimed, held = run_async(
_run_scheduler(
local_db, _harness(local_db, started, gate, peak), gate=gate
)()
)
assert held == 1
assert max(peak) == 1
assert len(claimed) == 1
done = local_db["devii_tasks"].find_one(uid=claimed[0])
assert int(done["run_count"]) == 1
assert done["status"] == "pending"
@@ -114,31 +128,131 @@ def test_scheduler_runs_one_task_per_owner_and_completes_it(local_db):
assert local_db["devii_tasks"].find_one(uid=other)["status"] == "pending"
def test_scheduler_retires_a_task_whose_owner_may_not_schedule(local_db):
def test_scheduler_runs_a_member_task(local_db):
local_db["devii_tasks"].delete()
uid = _seed(local_db, _account(local_db, "Member"))
owner = _account(local_db, "Member")
uid = _seed(local_db, owner)
started: list[str] = []
gate = asyncio.Event()
async def run():
gate = asyncio.Event()
gate.set()
scheduler = GlobalScheduler(
local_db, _harness(local_db, started, gate), tick_seconds=0.05
run_async(_run_scheduler(local_db, _harness(local_db, started, gate), gate=gate)())
assert started == [owner]
row = local_db["devii_tasks"].find_one(uid=uid)
assert row["enabled"]
assert int(row["run_count"]) == 1
def test_the_executor_runs_inside_the_task_run_context(local_db):
local_db["devii_tasks"].delete()
owner = _account(local_db, "Member")
_seed(local_db, owner)
started: list[str] = []
flags: list[bool] = []
gate = asyncio.Event()
run_async(
_run_scheduler(
local_db, _harness(local_db, started, gate, flags=flags), gate=gate
)()
)
assert flags == [True]
assert in_task_run() is False
def test_every_run_is_recorded_against_the_owner_quota(local_db):
local_db["devii_tasks"].delete()
local_db[limits.RUNS_TABLE].delete()
owner = _account(local_db, "Member")
_seed(local_db, owner)
started: list[str] = []
gate = asyncio.Event()
run_async(_run_scheduler(local_db, _harness(local_db, started, gate), gate=gate)())
quota = limits.run_quota(local_db, "user", owner, now_utc())
assert quota.used == 1
assert quota.limit == limits.DEFAULT_MEMBER_RUNS
def test_a_member_over_the_run_quota_is_postponed_not_disabled(local_db):
local_db["devii_tasks"].delete()
local_db[limits.RUNS_TABLE].delete()
owner = _account(local_db, "Member")
uid = _seed(local_db, owner)
now = now_utc()
for index in range(limits.DEFAULT_MEMBER_RUNS):
limits.record_run(
local_db, "user", owner, generate_uid(), now - timedelta(minutes=index)
)
scheduler.start()
await asyncio.sleep(0.3)
await scheduler.stop()
started: list[str] = []
gate = asyncio.Event()
run_async(run())
run_async(
_run_scheduler(local_db, _harness(local_db, started, gate), gate=gate)()
)
row = local_db["devii_tasks"].find_one(uid=uid)
assert started == []
assert row["enabled"]
assert row["status"] == "pending"
assert row["last_error"] == REASON_RUN_QUOTA
assert row["next_run_at"] > to_iso(now)
def test_an_administrator_keeps_running_past_the_member_limit(local_db):
local_db["devii_tasks"].delete()
local_db[limits.RUNS_TABLE].delete()
owner = _account(local_db, "Admin")
_seed(local_db, owner)
now = now_utc()
for index in range(limits.DEFAULT_MEMBER_RUNS + 5):
limits.record_run(
local_db, "user", owner, generate_uid(), now - timedelta(minutes=index)
)
started: list[str] = []
gate = asyncio.Event()
run_async(_run_scheduler(local_db, _harness(local_db, started, gate), gate=gate)())
assert started == [owner]
def test_budget_postpones_the_task(local_db):
local_db["devii_tasks"].delete()
local_db[limits.RUNS_TABLE].delete()
owner = _account(local_db, "Admin")
uid = _seed(local_db, owner)
started: list[str] = []
gate = asyncio.Event()
run_async(
_run_scheduler(
local_db,
_harness(local_db, started, gate),
gate=gate,
budget_exceeded=lambda kind, owner_id: True,
)()
)
row = local_db["devii_tasks"].find_one(uid=uid)
assert started == []
assert row["enabled"]
assert row["last_error"] == REASON_BUDGET
def test_scheduler_retires_a_guest_owned_task(local_db):
local_db["devii_tasks"].delete()
uid = _seed(local_db, "guest-cookie", owner_kind="guest")
started: list[str] = []
gate = asyncio.Event()
gate.set()
run_async(_run_scheduler(local_db, _harness(local_db, started, gate), gate=gate)())
row = local_db["devii_tasks"].find_one(uid=uid)
assert not row["enabled"]
assert row["status"] == "disabled"
assert row["last_error"] == REASON_NOT_ADMIN
assert row["last_error"] == REASON_NOT_A_USER
assert started == []
def test_scheduler_retires_expired_and_exhausted_tasks_even_while_saturated(local_db):
local_db["devii_tasks"].delete()
local_db[limits.RUNS_TABLE].delete()
owner = _account(local_db, "Admin")
_seed(local_db, owner)
expired = _seed(
@@ -146,27 +260,20 @@ def test_scheduler_retires_expired_and_exhausted_tasks_even_while_saturated(loca
)
exhausted = _seed(local_db, owner, max_runs=3, run_count=3)
started: list[str] = []
gate = asyncio.Event()
async def run():
gate = asyncio.Event()
scheduler = GlobalScheduler(
local_db, _harness(local_db, started, gate), tick_seconds=0.05
)
scheduler.start()
await asyncio.sleep(0.4)
gate.set()
await asyncio.sleep(0.2)
await scheduler.stop()
run_async(run())
run_async(_run_scheduler(local_db, _harness(local_db, started, gate), gate=gate)())
assert local_db["devii_tasks"].find_one(uid=expired)["last_error"] == REASON_EXPIRED
assert local_db["devii_tasks"].find_one(uid=exhausted)["last_error"] == REASON_MAX_RUNS
assert (
local_db["devii_tasks"].find_one(uid=exhausted)["last_error"] == REASON_MAX_RUNS
)
for uid in (expired, exhausted):
assert not local_db["devii_tasks"].find_one(uid=uid)["enabled"]
def test_scheduler_disables_a_task_after_repeated_failures(local_db):
local_db["devii_tasks"].delete()
local_db[limits.RUNS_TABLE].delete()
owner = _account(local_db, "Admin")
uid = _seed(local_db, owner, failure_count=2)
@@ -192,3 +299,27 @@ def test_scheduler_disables_a_task_after_repeated_failures(local_db):
assert not row["enabled"]
assert row["status"] == "error"
assert "upstream is down" in row["last_error"]
def test_a_failed_run_still_counts_against_the_quota(local_db):
local_db["devii_tasks"].delete()
local_db[limits.RUNS_TABLE].delete()
owner = _account(local_db, "Member")
_seed(local_db, owner)
def resolve(row):
store = TaskStore(local_db, "user", str(row["owner_id"]))
async def executor(prompt):
raise RuntimeError("nope")
return store, executor, lambda kind, task_row, payload: None
async def run():
scheduler = GlobalScheduler(local_db, resolve, tick_seconds=0.05)
scheduler.start()
await asyncio.sleep(0.3)
await scheduler.stop()
run_async(run())
assert limits.run_quota(local_db, "user", owner, now_utc()).used >= 1