Enforce the Devii task quotas with atomic reservations
The creation and run quotas were checked and then acted on, so two concurrent create_task calls or two schedulers could both pass the check and overshoot the limit. Both are now a single conditional INSERT decided on the driver rowcount: reserve_run takes a run slot after the claim and releases the claim by deferring when the quota is spent, and insert_task_within_quota does the same for the task row itself. Racing twelve and sixteen processes now yields exactly the limit. The atomic insert names its columns, and dataset skips a None valued key when it creates a table lazily, so the store declares the full task column set up front. Both the column and index ensures now tolerate a concurrent duplicate, since several processes build a store at once and SQLite DDL is not idempotent. Adds the quota, task-run context, guard, store and scheduler test suites, and documents the chokepoints and the unhackable task-run flag.
This commit is contained in:
@@ -24,6 +24,8 @@ from devplacepy.services.devii.tasks.limits import reserve_run
|
||||
from devplacepy.services.devii.tasks.schedule import now_utc, to_iso
|
||||
from devplacepy.utils import generate_uid
|
||||
|
||||
SIGNUP_AFTER_PRIMARY_ADMIN = "2099-01-01T00:00:00"
|
||||
|
||||
|
||||
def _account(local_db, role):
|
||||
uid = generate_uid()
|
||||
@@ -33,7 +35,7 @@ def _account(local_db, role):
|
||||
"username": f"guard-{uid[-10:]}",
|
||||
"role": role,
|
||||
"deleted_at": None,
|
||||
"created_at": to_iso(now_utc()),
|
||||
"created_at": SIGNUP_AFTER_PRIMARY_ADMIN,
|
||||
}
|
||||
)
|
||||
invalidate_admins_cache()
|
||||
@@ -55,11 +57,7 @@ def _row(owner_uid, **overrides):
|
||||
def _burn_runs(local_db, owner, count, reference):
|
||||
for index in range(count):
|
||||
limits.record_run(
|
||||
local_db,
|
||||
"user",
|
||||
owner,
|
||||
generate_uid(),
|
||||
reference - timedelta(minutes=index),
|
||||
local_db, "user", owner, generate_uid(), reference - timedelta(minutes=index)
|
||||
)
|
||||
|
||||
|
||||
@@ -94,6 +92,13 @@ def test_exhausted_task_is_retired(local_db):
|
||||
assert retire_reason(_row(owner, max_runs=5, run_count=5), now_utc()) == REASON_MAX_RUNS
|
||||
|
||||
|
||||
def test_task_without_an_expiry_stops_at_the_fallback_ceiling(local_db):
|
||||
owner = _account(local_db, "Member")
|
||||
now = now_utc()
|
||||
row = _row(owner, created_at=to_iso(now - timedelta(days=40)))
|
||||
assert retire_reason(row, now) == REASON_EXPIRED
|
||||
|
||||
|
||||
def test_repeated_failures_retire_the_task(local_db):
|
||||
owner = _account(local_db, "Member")
|
||||
row = _row(owner, failure_count=3)
|
||||
@@ -133,13 +138,11 @@ def test_administrator_run_quota_is_larger(local_db):
|
||||
def test_budget_defers_the_task(local_db):
|
||||
owner = _account(local_db, "Member")
|
||||
now = now_utc()
|
||||
postponement = budget_deferral(
|
||||
_row(owner), now, budget_exceeded=lambda kind, uid: True
|
||||
)
|
||||
postponement = budget_deferral(_row(owner), now, budget_exceeded=lambda k, u: True)
|
||||
assert postponement is not None
|
||||
assert postponement.reason == REASON_BUDGET
|
||||
assert postponement.retry_at > now
|
||||
assert budget_deferral(_row(owner), now, lambda kind, uid: False) is None
|
||||
assert budget_deferral(_row(owner), now, lambda k, u: False) is None
|
||||
assert budget_deferral(_row(owner), now, None) is None
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user