Quiiz system
This commit is contained in:
@@ -119,3 +119,32 @@ def test_game_actions_registered_with_correct_auth():
|
||||
def test_game_read_actions_are_read_only():
|
||||
for name in ("game_state", "game_leaderboard", "game_view_farm"):
|
||||
assert BY_NAME[name].is_read_only is True
|
||||
|
||||
|
||||
def test_scheduling_tools_are_administrator_only():
|
||||
from devplacepy.services.devii.registry import CATALOG
|
||||
|
||||
by_name = CATALOG.by_name()
|
||||
for name in ("create_task", "update_task", "run_task_now"):
|
||||
action = by_name[name]
|
||||
assert action.requires_admin is True
|
||||
assert action.requires_auth is True
|
||||
for name in ("list_tasks", "get_task", "delete_task"):
|
||||
assert by_name[name].requires_admin is False
|
||||
|
||||
|
||||
def test_scheduling_tools_hidden_from_a_member_tool_list():
|
||||
from devplacepy.services.devii.registry import CATALOG
|
||||
|
||||
member = {
|
||||
s["function"]["name"]
|
||||
for s in CATALOG.tool_schemas_for(authenticated=True, is_admin=False)
|
||||
}
|
||||
admin = {
|
||||
s["function"]["name"]
|
||||
for s in CATALOG.tool_schemas_for(authenticated=True, is_admin=True)
|
||||
}
|
||||
for name in ("create_task", "update_task", "run_task_now"):
|
||||
assert name not in member
|
||||
assert name in admin
|
||||
assert "list_tasks" in member
|
||||
|
||||
@@ -71,10 +71,32 @@ def test_guest_main_has_ui_prompt_without_pref_tools(local_db):
|
||||
assert "interactions_set" not in names
|
||||
|
||||
|
||||
def test_docs_channel_does_not_start_scheduler(local_db):
|
||||
docs = _session("docs", "sess-docs-sched")
|
||||
docs.attach(object())
|
||||
assert docs._started is False
|
||||
def test_scheduled_tasks_only_ever_resolve_a_main_channel_session(local_db):
|
||||
hub, svc = _hub()
|
||||
channels = []
|
||||
original = hub.get_or_create
|
||||
|
||||
def capture(*args, **kwargs):
|
||||
channels.append(kwargs.get("channel"))
|
||||
return original(*args, **kwargs)
|
||||
|
||||
hub.get_or_create = capture
|
||||
try:
|
||||
row = {"owner_kind": "user", "owner_id": "sess-docs-sched"}
|
||||
local_db["users"].insert(
|
||||
{
|
||||
"uid": "sess-docs-sched",
|
||||
"username": "sched-owner",
|
||||
"role": "Admin",
|
||||
"api_key": "k",
|
||||
"deleted_at": None,
|
||||
}
|
||||
)
|
||||
database.invalidate_admins_cache()
|
||||
svc._resolve_task_owner(row)
|
||||
finally:
|
||||
hub.get_or_create = original
|
||||
assert channels == ["main"]
|
||||
|
||||
|
||||
def test_user_docs_channel_uses_ephemeral_stores(local_db):
|
||||
|
||||
@@ -0,0 +1,95 @@
|
||||
# retoor <retoor@molodetz.nl>
|
||||
|
||||
from datetime import timedelta
|
||||
|
||||
from devplacepy.database import invalidate_admins_cache
|
||||
from devplacepy.services.devii.tasks.guards import (
|
||||
REASON_BUDGET,
|
||||
REASON_EXPIRED,
|
||||
REASON_FAILURES,
|
||||
REASON_MAX_RUNS,
|
||||
REASON_NOT_ADMIN,
|
||||
automation_allowed,
|
||||
refusal,
|
||||
)
|
||||
from devplacepy.services.devii.tasks.schedule import now_utc, to_iso
|
||||
from devplacepy.utils import generate_uid
|
||||
|
||||
|
||||
def _account(local_db, role):
|
||||
uid = generate_uid()
|
||||
local_db["users"].insert(
|
||||
{
|
||||
"uid": uid,
|
||||
"username": f"guard-{uid[-10:]}",
|
||||
"role": role,
|
||||
"deleted_at": None,
|
||||
}
|
||||
)
|
||||
invalidate_admins_cache()
|
||||
return uid
|
||||
|
||||
|
||||
def _row(owner_uid, **overrides):
|
||||
row = {
|
||||
"uid": generate_uid(),
|
||||
"owner_kind": "user",
|
||||
"owner_id": owner_uid,
|
||||
"run_count": 0,
|
||||
"created_at": to_iso(now_utc()),
|
||||
}
|
||||
row.update(overrides)
|
||||
return row
|
||||
|
||||
|
||||
def test_only_administrators_may_automate(local_db):
|
||||
admin = _account(local_db, "Admin")
|
||||
member = _account(local_db, "Member")
|
||||
assert automation_allowed("user", admin) is True
|
||||
assert automation_allowed("user", member) is False
|
||||
assert automation_allowed("guest", "guest-cookie") is False
|
||||
assert automation_allowed("user", "") is False
|
||||
|
||||
|
||||
def test_member_owned_task_is_refused(local_db):
|
||||
member = _account(local_db, "Member")
|
||||
assert refusal(_row(member), now_utc()) == REASON_NOT_ADMIN
|
||||
|
||||
|
||||
def test_clean_admin_task_is_allowed(local_db):
|
||||
admin = _account(local_db, "Admin")
|
||||
assert refusal(_row(admin), now_utc()) is None
|
||||
|
||||
|
||||
def test_exhausted_task_is_refused(local_db):
|
||||
admin = _account(local_db, "Admin")
|
||||
row = _row(admin, max_runs=5, run_count=5)
|
||||
assert refusal(row, now_utc()) == REASON_MAX_RUNS
|
||||
|
||||
|
||||
def test_expired_task_is_refused(local_db):
|
||||
admin = _account(local_db, "Admin")
|
||||
now = now_utc()
|
||||
row = _row(admin, expires_at=to_iso(now - timedelta(minutes=1)))
|
||||
assert refusal(row, now) == REASON_EXPIRED
|
||||
|
||||
|
||||
def test_task_without_an_expiry_stops_at_the_fallback_ceiling(local_db):
|
||||
admin = _account(local_db, "Admin")
|
||||
now = now_utc()
|
||||
row = _row(admin, created_at=to_iso(now - timedelta(days=40)))
|
||||
assert refusal(row, now) == REASON_EXPIRED
|
||||
|
||||
|
||||
def test_repeated_failures_refuse_the_task(local_db):
|
||||
admin = _account(local_db, "Admin")
|
||||
row = _row(admin, failure_count=3)
|
||||
assert refusal(row, now_utc(), max_failures=3) == REASON_FAILURES
|
||||
assert refusal(row, now_utc(), max_failures=0) is None
|
||||
|
||||
|
||||
def test_budget_probe_refuses_the_task(local_db):
|
||||
admin = _account(local_db, "Admin")
|
||||
row = _row(admin)
|
||||
assert refusal(row, now_utc(), budget_exceeded=lambda k, i: True) == REASON_BUDGET
|
||||
assert refusal(row, now_utc(), budget_exceeded=lambda k, i: False) is None
|
||||
@@ -0,0 +1,79 @@
|
||||
# retoor <retoor@molodetz.nl>
|
||||
|
||||
import pytest
|
||||
|
||||
from devplacepy.services.devii.tasks.guards import task_columns
|
||||
from devplacepy.services.devii.tasks.schedule import (
|
||||
DEFAULT_MAX_RUNS,
|
||||
MAX_LIFETIME_DAYS,
|
||||
MAX_MAX_RUNS,
|
||||
MIN_INTERVAL_SECONDS,
|
||||
Schedule,
|
||||
from_iso,
|
||||
now_utc,
|
||||
)
|
||||
|
||||
|
||||
def test_interval_below_the_floor_is_rejected():
|
||||
with pytest.raises(ValueError):
|
||||
Schedule(kind="interval", every_seconds=MIN_INTERVAL_SECONDS - 1)
|
||||
with pytest.raises(ValueError):
|
||||
Schedule(kind="interval", every_seconds=180)
|
||||
|
||||
|
||||
def test_interval_at_the_floor_is_accepted():
|
||||
schedule = Schedule(kind="interval", every_seconds=MIN_INTERVAL_SECONDS)
|
||||
assert schedule.every_seconds == MIN_INTERVAL_SECONDS
|
||||
|
||||
|
||||
def test_cron_firing_faster_than_the_floor_is_rejected():
|
||||
for expression in ("* * * * *", "*/5 * * * *", "*/14 * * * *"):
|
||||
with pytest.raises(ValueError):
|
||||
Schedule(kind="cron", cron=expression)
|
||||
|
||||
|
||||
def test_cron_with_an_uneven_step_is_rejected_for_its_boundary_hop():
|
||||
with pytest.raises(ValueError):
|
||||
Schedule(kind="cron", cron="*/18 * * * *")
|
||||
|
||||
|
||||
def test_cron_at_or_above_the_floor_is_accepted():
|
||||
for expression in ("*/15 * * * *", "0 * * * *", "0 22 * * *"):
|
||||
assert Schedule(kind="cron", cron=expression).cron == expression
|
||||
|
||||
|
||||
def test_max_runs_ceiling_is_enforced():
|
||||
with pytest.raises(ValueError):
|
||||
Schedule(kind="interval", every_seconds=900, max_runs=MAX_MAX_RUNS + 1)
|
||||
with pytest.raises(ValueError):
|
||||
Schedule(kind="interval", every_seconds=900, max_runs=0)
|
||||
|
||||
|
||||
def test_recurring_task_without_max_runs_gets_the_default():
|
||||
reference = now_utc()
|
||||
columns = task_columns(Schedule(kind="interval", every_seconds=900), reference)
|
||||
assert columns["max_runs"] == DEFAULT_MAX_RUNS
|
||||
|
||||
|
||||
def test_one_shot_task_keeps_no_run_limit():
|
||||
reference = now_utc()
|
||||
columns = task_columns(Schedule(kind="once", delay_seconds=60), reference)
|
||||
assert columns["max_runs"] is None
|
||||
|
||||
|
||||
def test_every_task_gets_an_expiry_within_the_ceiling():
|
||||
reference = now_utc()
|
||||
columns = task_columns(Schedule(kind="interval", every_seconds=900), reference)
|
||||
expiry = from_iso(columns["expires_at"])
|
||||
assert expiry > reference
|
||||
assert (expiry - reference).days <= MAX_LIFETIME_DAYS
|
||||
|
||||
|
||||
def test_a_caller_cannot_extend_the_expiry_past_the_ceiling():
|
||||
reference = now_utc()
|
||||
far = reference.replace(year=reference.year + 5)
|
||||
columns = task_columns(
|
||||
Schedule(kind="interval", every_seconds=900, expires_at=far), reference
|
||||
)
|
||||
expiry = from_iso(columns["expires_at"])
|
||||
assert (expiry - reference).days <= MAX_LIFETIME_DAYS
|
||||
@@ -0,0 +1,194 @@
|
||||
# retoor <retoor@molodetz.nl>
|
||||
|
||||
import asyncio
|
||||
|
||||
from devplacepy.database import invalidate_admins_cache
|
||||
from devplacepy.services.devii.tasks.guards import (
|
||||
REASON_EXPIRED,
|
||||
REASON_MAX_RUNS,
|
||||
REASON_NOT_ADMIN,
|
||||
)
|
||||
from devplacepy.services.devii.tasks.schedule import now_utc, to_iso
|
||||
from devplacepy.services.devii.tasks.scheduler import GlobalScheduler
|
||||
from devplacepy.services.devii.tasks.store import TaskStore
|
||||
from devplacepy.utils import generate_uid
|
||||
from tests.conftest import run_async
|
||||
|
||||
|
||||
def _account(local_db, role):
|
||||
uid = generate_uid()
|
||||
local_db["users"].insert(
|
||||
{
|
||||
"uid": uid,
|
||||
"username": f"sched-{uid[-10:]}",
|
||||
"role": role,
|
||||
"api_key": "k",
|
||||
"deleted_at": None,
|
||||
}
|
||||
)
|
||||
invalidate_admins_cache()
|
||||
return uid
|
||||
|
||||
|
||||
def _seed(local_db, owner, **overrides):
|
||||
row = {
|
||||
"uid": generate_uid(),
|
||||
"owner_kind": "user",
|
||||
"owner_id": owner,
|
||||
"label": "scheduled",
|
||||
"prompt": "work",
|
||||
"enabled": True,
|
||||
"status": "pending",
|
||||
"kind": "interval",
|
||||
"every_seconds": 900,
|
||||
"max_runs": 10,
|
||||
"run_count": 0,
|
||||
"failure_count": 0,
|
||||
"created_at": to_iso(now_utc()),
|
||||
"next_run_at": to_iso(now_utc()),
|
||||
"expires_at": None,
|
||||
"deleted_at": None,
|
||||
"deleted_by": None,
|
||||
}
|
||||
row.update(overrides)
|
||||
local_db["devii_tasks"].insert(row)
|
||||
return row["uid"]
|
||||
|
||||
|
||||
def _harness(local_db, started, gate, peak=None):
|
||||
active: dict[str, int] = {}
|
||||
|
||||
def resolve(row):
|
||||
owner = str(row["owner_id"])
|
||||
store = TaskStore(local_db, "user", owner)
|
||||
|
||||
async def executor(prompt):
|
||||
started.append(owner)
|
||||
active[owner] = active.get(owner, 0) + 1
|
||||
if peak is not None:
|
||||
peak.append(max(active.values()))
|
||||
try:
|
||||
await gate.wait()
|
||||
return f"done: {prompt}"
|
||||
finally:
|
||||
active[owner] -= 1
|
||||
|
||||
return store, executor, lambda kind, task_row, payload: None
|
||||
|
||||
return resolve
|
||||
|
||||
|
||||
def test_scheduler_runs_one_task_per_owner_and_completes_it(local_db):
|
||||
local_db["devii_tasks"].delete()
|
||||
owner = _account(local_db, "Admin")
|
||||
first = _seed(local_db, owner)
|
||||
second = _seed(local_db, owner)
|
||||
|
||||
started: list[str] = []
|
||||
peak: list[int] = []
|
||||
|
||||
async def run():
|
||||
gate = asyncio.Event()
|
||||
scheduler = GlobalScheduler(
|
||||
local_db, _harness(local_db, started, gate, peak), tick_seconds=0.05
|
||||
)
|
||||
scheduler.start()
|
||||
await asyncio.sleep(0.4)
|
||||
claimed = [r["uid"] for r in local_db["devii_tasks"].find(status="running")]
|
||||
held = len(started)
|
||||
gate.set()
|
||||
await asyncio.sleep(0.5)
|
||||
await scheduler.stop()
|
||||
return claimed, held
|
||||
|
||||
claimed, held = run_async(run())
|
||||
assert held == 1
|
||||
assert max(peak) == 1
|
||||
assert len(claimed) == 1
|
||||
done = local_db["devii_tasks"].find_one(uid=claimed[0])
|
||||
assert int(done["run_count"]) == 1
|
||||
assert done["status"] == "pending"
|
||||
assert done["last_result"].startswith("done:")
|
||||
assert done["next_run_at"] > to_iso(now_utc())
|
||||
other = second if claimed[0] == first else first
|
||||
assert local_db["devii_tasks"].find_one(uid=other)["status"] == "pending"
|
||||
|
||||
|
||||
def test_scheduler_retires_a_task_whose_owner_may_not_schedule(local_db):
|
||||
local_db["devii_tasks"].delete()
|
||||
uid = _seed(local_db, _account(local_db, "Member"))
|
||||
started: list[str] = []
|
||||
|
||||
async def run():
|
||||
gate = asyncio.Event()
|
||||
gate.set()
|
||||
scheduler = GlobalScheduler(
|
||||
local_db, _harness(local_db, started, gate), tick_seconds=0.05
|
||||
)
|
||||
scheduler.start()
|
||||
await asyncio.sleep(0.3)
|
||||
await scheduler.stop()
|
||||
|
||||
run_async(run())
|
||||
row = local_db["devii_tasks"].find_one(uid=uid)
|
||||
assert not row["enabled"]
|
||||
assert row["status"] == "disabled"
|
||||
assert row["last_error"] == REASON_NOT_ADMIN
|
||||
assert started == []
|
||||
|
||||
|
||||
def test_scheduler_retires_expired_and_exhausted_tasks_even_while_saturated(local_db):
|
||||
local_db["devii_tasks"].delete()
|
||||
owner = _account(local_db, "Admin")
|
||||
_seed(local_db, owner)
|
||||
expired = _seed(
|
||||
local_db, owner, expires_at=to_iso(now_utc().replace(year=now_utc().year - 1))
|
||||
)
|
||||
exhausted = _seed(local_db, owner, max_runs=3, run_count=3)
|
||||
started: list[str] = []
|
||||
|
||||
async def run():
|
||||
gate = asyncio.Event()
|
||||
scheduler = GlobalScheduler(
|
||||
local_db, _harness(local_db, started, gate), tick_seconds=0.05
|
||||
)
|
||||
scheduler.start()
|
||||
await asyncio.sleep(0.4)
|
||||
gate.set()
|
||||
await asyncio.sleep(0.2)
|
||||
await scheduler.stop()
|
||||
|
||||
run_async(run())
|
||||
assert local_db["devii_tasks"].find_one(uid=expired)["last_error"] == REASON_EXPIRED
|
||||
assert local_db["devii_tasks"].find_one(uid=exhausted)["last_error"] == REASON_MAX_RUNS
|
||||
for uid in (expired, exhausted):
|
||||
assert not local_db["devii_tasks"].find_one(uid=uid)["enabled"]
|
||||
|
||||
|
||||
def test_scheduler_disables_a_task_after_repeated_failures(local_db):
|
||||
local_db["devii_tasks"].delete()
|
||||
owner = _account(local_db, "Admin")
|
||||
uid = _seed(local_db, owner, failure_count=2)
|
||||
|
||||
def resolve(row):
|
||||
store = TaskStore(local_db, "user", str(row["owner_id"]))
|
||||
|
||||
async def executor(prompt):
|
||||
raise RuntimeError("upstream is down")
|
||||
|
||||
return store, executor, lambda kind, task_row, payload: None
|
||||
|
||||
async def run():
|
||||
scheduler = GlobalScheduler(
|
||||
local_db, resolve, tick_seconds=0.05, max_failures=3
|
||||
)
|
||||
scheduler.start()
|
||||
await asyncio.sleep(0.4)
|
||||
await scheduler.stop()
|
||||
|
||||
run_async(run())
|
||||
row = local_db["devii_tasks"].find_one(uid=uid)
|
||||
assert int(row["failure_count"]) == 3
|
||||
assert not row["enabled"]
|
||||
assert row["status"] == "error"
|
||||
assert "upstream is down" in row["last_error"]
|
||||
@@ -0,0 +1,136 @@
|
||||
# retoor <retoor@molodetz.nl>
|
||||
|
||||
import pytest
|
||||
|
||||
from devplacepy.database import invalidate_admins_cache
|
||||
from devplacepy.services.devii.errors import ToolInputError
|
||||
from devplacepy.services.devii.tasks.controller import TaskController
|
||||
from devplacepy.services.devii.tasks.guards import AutomationDenied
|
||||
from devplacepy.services.devii.tasks.schedule import now_utc, to_iso
|
||||
from devplacepy.services.devii.tasks.store import TaskStore, claim, due_rows
|
||||
from devplacepy.utils import generate_uid
|
||||
|
||||
|
||||
def _account(local_db, role):
|
||||
uid = generate_uid()
|
||||
local_db["users"].insert(
|
||||
{
|
||||
"uid": uid,
|
||||
"username": f"store-{uid[-10:]}",
|
||||
"role": role,
|
||||
"deleted_at": None,
|
||||
}
|
||||
)
|
||||
invalidate_admins_cache()
|
||||
return uid
|
||||
|
||||
|
||||
def _record(**overrides):
|
||||
record = {
|
||||
"uid": generate_uid(),
|
||||
"prompt": "work",
|
||||
"enabled": True,
|
||||
"status": "pending",
|
||||
"created_at": to_iso(now_utc()),
|
||||
"next_run_at": to_iso(now_utc()),
|
||||
"run_count": 0,
|
||||
"failure_count": 0,
|
||||
"kind": "interval",
|
||||
"every_seconds": 900,
|
||||
"max_runs": 10,
|
||||
}
|
||||
record.update(overrides)
|
||||
return record
|
||||
|
||||
|
||||
def test_member_cannot_persist_a_task(local_db):
|
||||
store = TaskStore(local_db, "user", _account(local_db, "Member"))
|
||||
with pytest.raises(AutomationDenied):
|
||||
store.create(_record())
|
||||
|
||||
|
||||
def test_administrator_can_persist_a_task(local_db):
|
||||
store = TaskStore(local_db, "user", _account(local_db, "Admin"))
|
||||
record = _record()
|
||||
store.create(record)
|
||||
assert store.get(record["uid"]) is not None
|
||||
|
||||
|
||||
def test_member_cannot_re_enable_a_task(local_db):
|
||||
admin = _account(local_db, "Admin")
|
||||
member = _account(local_db, "Member")
|
||||
admin_store = TaskStore(local_db, "user", admin)
|
||||
record = _record()
|
||||
admin_store.create(record)
|
||||
local_db["devii_tasks"].update(
|
||||
{"uid": record["uid"], "owner_id": member, "enabled": False}, ["uid"]
|
||||
)
|
||||
member_store = TaskStore(local_db, "user", member)
|
||||
with pytest.raises(AutomationDenied):
|
||||
member_store.update(record["uid"], {"enabled": True})
|
||||
member_store.update(record["uid"], {"enabled": False, "status": "disabled"})
|
||||
|
||||
|
||||
def test_local_operator_store_bypasses_the_role_gate(local_db):
|
||||
store = TaskStore(local_db, "user", "cli", operator=True)
|
||||
record = _record()
|
||||
store.create(record)
|
||||
assert store.get(record["uid"]) is not None
|
||||
|
||||
|
||||
def test_claim_succeeds_once(local_db):
|
||||
store = TaskStore(local_db, "user", _account(local_db, "Admin"))
|
||||
record = _record()
|
||||
store.create(record)
|
||||
assert claim(local_db, record["uid"]) is True
|
||||
assert claim(local_db, record["uid"]) is False
|
||||
|
||||
|
||||
def test_claim_refuses_a_disabled_task(local_db):
|
||||
store = TaskStore(local_db, "user", _account(local_db, "Admin"))
|
||||
record = _record()
|
||||
store.create(record)
|
||||
store.update(record["uid"], {"enabled": False, "status": "disabled"})
|
||||
assert claim(local_db, record["uid"]) is False
|
||||
|
||||
|
||||
def test_due_rows_skips_future_and_running_tasks(local_db):
|
||||
admin = _account(local_db, "Admin")
|
||||
store = TaskStore(local_db, "user", admin)
|
||||
now = now_utc()
|
||||
due = _record(next_run_at=to_iso(now.replace(year=now.year - 1)))
|
||||
future = _record(next_run_at=to_iso(now.replace(year=now.year + 1)))
|
||||
running = _record(
|
||||
status="running", next_run_at=to_iso(now.replace(year=now.year - 1))
|
||||
)
|
||||
for record in (due, future, running):
|
||||
store.create(record)
|
||||
found = {row["uid"] for row in due_rows(local_db, to_iso(now), 500)}
|
||||
assert due["uid"] in found
|
||||
assert future["uid"] not in found
|
||||
assert running["uid"] not in found
|
||||
|
||||
|
||||
def test_controller_caps_active_tasks_per_owner(local_db, monkeypatch):
|
||||
monkeypatch.setattr(
|
||||
"devplacepy.services.devii.tasks.controller.max_active_per_owner", lambda: 2
|
||||
)
|
||||
store = TaskStore(local_db, "user", _account(local_db, "Admin"))
|
||||
controller = TaskController(store)
|
||||
for _ in range(2):
|
||||
controller.create_task(
|
||||
{"prompt": "work", "kind": "interval", "every_seconds": 900}
|
||||
)
|
||||
with pytest.raises(ToolInputError):
|
||||
controller.create_task(
|
||||
{"prompt": "work", "kind": "interval", "every_seconds": 900}
|
||||
)
|
||||
|
||||
|
||||
def test_controller_refuses_a_member(local_db):
|
||||
store = TaskStore(local_db, "user", _account(local_db, "Member"))
|
||||
controller = TaskController(store)
|
||||
with pytest.raises(ToolInputError):
|
||||
controller.create_task(
|
||||
{"prompt": "work", "kind": "interval", "every_seconds": 900}
|
||||
)
|
||||
@@ -270,7 +270,7 @@ def test_market_buff_factor_only_applies_to_starter_crops():
|
||||
|
||||
def test_market_factor_bounds_across_domain():
|
||||
floor = economy.MARKET_SATURATION_TIERS[-1][1]
|
||||
for key in economy.MARKET_TRACKED_CROPS:
|
||||
for key in economy.MARKET_BUFFED_CROPS + economy.MARKET_PRESSURE_CROPS:
|
||||
crop = economy.crop_for(key)
|
||||
for harvests in range(0, 300000, 977):
|
||||
saturation = economy.market_saturation_factor(economy.supply_days(crop, harvests))
|
||||
@@ -365,11 +365,13 @@ def test_cosmetic_title_name_only_resolves_titles():
|
||||
|
||||
|
||||
def test_mastery_points_awarded_first_point_at_unlock():
|
||||
assert economy.mastery_points_awarded(49, 50) == 1
|
||||
assert economy.mastery_points_awarded(0, 49) == 0
|
||||
assert economy.mastery_points_awarded(55, 56) == 0
|
||||
assert economy.mastery_points_awarded(59, 60) == 1
|
||||
assert economy.mastery_points_awarded(50, 70) == 2
|
||||
step = economy.MASTERY_PRESTIGE_STEP
|
||||
unlock = economy.MASTERY_UNLOCK_PRESTIGE
|
||||
assert economy.mastery_points_awarded(unlock - 1, unlock) == 1
|
||||
assert economy.mastery_points_awarded(0, unlock - 1) == 0
|
||||
assert economy.mastery_points_awarded(unlock, unlock + step - 1) == 0
|
||||
assert economy.mastery_points_awarded(unlock + step - 1, unlock + step) == 1
|
||||
assert economy.mastery_points_awarded(unlock, unlock + step * 2) == 2
|
||||
|
||||
|
||||
def test_mastery_for_known_and_unknown():
|
||||
@@ -490,12 +492,182 @@ def test_refactor_carryover_zero_when_unaffordable():
|
||||
def test_grant_amount_capped_and_bounded():
|
||||
assert economy.grant_amount(0) == 0
|
||||
assert economy.grant_amount(-10) == 0
|
||||
assert economy.grant_amount(100) == 100
|
||||
assert economy.grant_amount(100) == 0
|
||||
assert economy.grant_amount(economy.GRANT_MIN_AMOUNT) == economy.GRANT_MIN_AMOUNT
|
||||
assert economy.grant_amount(10**9) == economy.GRANT_CAP
|
||||
|
||||
|
||||
def test_grant_amount_shares_between_eligible_farms():
|
||||
assert economy.grant_amount(10_000, 5) == 2_000
|
||||
assert economy.grant_amount(10**9, 4) == economy.GRANT_CAP
|
||||
assert economy.grant_amount(1_000, 10) == 0
|
||||
for eligible in range(1, 50):
|
||||
share = economy.grant_amount(10**6, eligible)
|
||||
assert 0 <= share <= economy.GRANT_CAP
|
||||
assert share * eligible <= 10**6 or share == economy.GRANT_CAP
|
||||
|
||||
|
||||
def test_legacy_carryover_upgrade_registered():
|
||||
upgrade = economy.legacy_for("carryover")
|
||||
assert upgrade is not None
|
||||
assert upgrade.max_level == 5
|
||||
assert upgrade.max_level == 10
|
||||
assert "carry-over" in economy.legacy_value_text(upgrade, 2)
|
||||
assert economy.refactor_carryover_fraction(upgrade.max_level) < 1.0
|
||||
|
||||
|
||||
def test_fertilize_is_never_profitable_across_the_input_domain():
|
||||
import itertools
|
||||
|
||||
for crop in economy.CROPS:
|
||||
for prestige in range(0, 201, 23):
|
||||
for yield_level in (0, 5, 10):
|
||||
for legacy in (0, 5, 10):
|
||||
for market in (0.40, 0.70, 1.0, 1.15):
|
||||
for golden, contract, underdog in itertools.product(
|
||||
(False, True), repeat=3
|
||||
):
|
||||
value = economy.realizable_harvest_coins(
|
||||
crop,
|
||||
yield_level,
|
||||
prestige,
|
||||
legacy,
|
||||
market,
|
||||
golden,
|
||||
contract,
|
||||
underdog,
|
||||
)
|
||||
cost = economy.fertilize_click_cost(
|
||||
value, crop.grow_seconds, crop.grow_seconds
|
||||
)
|
||||
assert cost >= value, (
|
||||
crop.key,
|
||||
prestige,
|
||||
golden,
|
||||
contract,
|
||||
underdog,
|
||||
cost,
|
||||
value,
|
||||
)
|
||||
|
||||
|
||||
def test_fertilize_priced_above_the_expected_canary_payout():
|
||||
for crop in economy.CROPS:
|
||||
for prestige in (0, 25, 100):
|
||||
plain = economy.realizable_harvest_coins(crop, 10, prestige, 10, 1.0)
|
||||
priced = economy.realizable_harvest_coins(
|
||||
crop, 10, prestige, 10, 1.0, canary=True
|
||||
)
|
||||
cost = economy.fertilize_click_cost(
|
||||
priced, crop.grow_seconds, crop.grow_seconds
|
||||
)
|
||||
expected = plain * (
|
||||
economy.CANARY_DOUBLE_CHANCE * 2
|
||||
+ (1 - economy.CANARY_DOUBLE_CHANCE - economy.CANARY_FAIL_CHANCE)
|
||||
) + economy.CANARY_FAIL_CHANCE * min(plain, crop.cost)
|
||||
assert cost >= expected
|
||||
|
||||
|
||||
def test_staged_fertilize_never_undercuts_the_payout():
|
||||
for key in ("shell", "api", "kernel", "secfort"):
|
||||
crop = economy.crop_for(key)
|
||||
value = economy.realizable_harvest_coins(crop, 10, 50, 10, 1.0, True, True, True)
|
||||
remaining, total = crop.grow_seconds, 0
|
||||
while True:
|
||||
reduce_by = int(remaining * economy.FERTILIZE_FRACTION)
|
||||
if reduce_by < 1:
|
||||
break
|
||||
total += economy.fertilize_click_cost(value, reduce_by, crop.grow_seconds)
|
||||
remaining -= reduce_by
|
||||
assert total >= value
|
||||
|
||||
|
||||
def test_every_defense_tier_reduces_the_raider_share():
|
||||
shares = []
|
||||
for tier in economy.DEFENSE_TIERS:
|
||||
share = economy.effective_steal_fraction(
|
||||
0, tier.steal_fraction_floor, tier.steal_reduction
|
||||
)
|
||||
shares.append(round(share, 4))
|
||||
assert len(set(shares)) == len(economy.DEFENSE_TIERS)
|
||||
assert shares == sorted(shares, reverse=True)
|
||||
|
||||
|
||||
def test_more_defense_never_increases_the_raider_share():
|
||||
previous = 1.0
|
||||
for tier in economy.DEFENSE_TIERS:
|
||||
for legacy in range(0, 6):
|
||||
share = economy.effective_steal_fraction(
|
||||
legacy, tier.steal_fraction_floor, tier.steal_reduction
|
||||
)
|
||||
assert 0.0 <= share <= 1.0
|
||||
top = economy.effective_steal_fraction(
|
||||
0, tier.steal_fraction_floor, tier.steal_reduction
|
||||
)
|
||||
assert top <= previous + 1e-9
|
||||
previous = top
|
||||
|
||||
|
||||
def test_observability_only_ever_lowers_the_raider_share():
|
||||
for tier in economy.DEFENSE_TIERS:
|
||||
for legacy in range(0, 6):
|
||||
without = economy.effective_steal_fraction(
|
||||
legacy, tier.steal_fraction_floor, tier.steal_reduction
|
||||
)
|
||||
with_suite = economy.effective_steal_fraction(
|
||||
legacy,
|
||||
tier.steal_fraction_floor,
|
||||
tier.steal_reduction,
|
||||
economy.OBSERVABILITY_STEAL_CAP,
|
||||
)
|
||||
assert with_suite <= without + 1e-9
|
||||
assert with_suite <= economy.OBSERVABILITY_STEAL_CAP + 1e-9
|
||||
|
||||
|
||||
def test_raid_share_plus_owner_remainder_never_exceeds_the_build():
|
||||
for crop in economy.CROPS:
|
||||
for prestige in range(0, 101, 17):
|
||||
for golden in (False, True):
|
||||
value = economy.realizable_harvest_coins(
|
||||
crop, 10, prestige, 10, 1.0, golden
|
||||
)
|
||||
for tier in economy.DEFENSE_TIERS:
|
||||
for legacy in range(0, 6):
|
||||
share = economy.effective_steal_fraction(
|
||||
legacy, tier.steal_fraction_floor, tier.steal_reduction
|
||||
)
|
||||
thief = min(max(1, round(value * share)), int(value * (1.0 - 0.0)))
|
||||
owner = int(value * (1.0 - share))
|
||||
assert thief + owner <= value
|
||||
|
||||
|
||||
def test_weekly_contract_stars_are_per_kind_not_per_goal():
|
||||
for kind in economy.QUEST_KINDS:
|
||||
for scale in (1.0, 13.5, 40.0):
|
||||
contract = economy.weekly_contract("user-a", "2026-W30", scale)
|
||||
if contract["kind"] != kind:
|
||||
continue
|
||||
assert contract["reward_stars"] == economy.QUEST_DEFS[kind].contract_stars
|
||||
assert contract["reward_stars"] <= 5
|
||||
|
||||
|
||||
def test_social_rewards_scale_with_the_earner_and_never_regress():
|
||||
assert economy.water_reward_coins(0, 0) == economy.WATER_REWARD_COINS
|
||||
assert economy.daily_reward(1) == economy.DAILY_BASE
|
||||
previous = 0
|
||||
for prestige in range(0, 101):
|
||||
reward = economy.water_reward_coins(prestige, 0)
|
||||
assert reward >= previous
|
||||
previous = reward
|
||||
|
||||
|
||||
def test_leaderboard_coin_contribution_is_capped():
|
||||
base = {"xp": 18050, "prestige": 50, "total_harvests": 5000, "ci_tier": 5}
|
||||
poor = economy.farm_score({**base, "coins": 0})
|
||||
rich = economy.farm_score({**base, "coins": 10**9})
|
||||
assert rich - poor <= economy.SCORE_COIN_CAP
|
||||
|
||||
|
||||
def test_market_saturation_is_per_capita():
|
||||
crop = economy.crop_for("kernel")
|
||||
assert economy.supply_days(crop, 288, 1) == economy.supply_days(crop, 2880, 10)
|
||||
assert economy.supply_days(crop, 288, 4) < economy.supply_days(crop, 288, 1)
|
||||
|
||||
@@ -195,7 +195,9 @@ def test_yield_perk_and_prestige_boost_harvest_coins(local_db):
|
||||
_warp(user)
|
||||
result = store.harvest(user, 0)
|
||||
crop = economy.crop_for("shell")
|
||||
assert result["coins"] == economy.effective_reward_coins(crop, 4, 1)
|
||||
assert result["coins"] == economy.realizable_harvest_coins(
|
||||
crop, 4, 1, golden=result["golden"]
|
||||
)
|
||||
assert result["coins"] > crop.reward_coins
|
||||
|
||||
|
||||
@@ -599,7 +601,7 @@ def _ripen_past_grace(user, slot=0):
|
||||
)
|
||||
|
||||
|
||||
def test_steal_transfers_coins_and_clears_plot(local_db):
|
||||
def test_steal_transfers_a_share_and_leaves_the_build(local_db):
|
||||
owner = _reset("unit_owner")
|
||||
thief = _reset("unit_thief", coins=0)
|
||||
store.plant(owner, 0, "shell")
|
||||
@@ -608,7 +610,57 @@ def test_steal_transfers_coins_and_clears_plot(local_db):
|
||||
assert result["coins"] > 0
|
||||
assert _coins(thief) == result["coins"]
|
||||
farm = store.get_farm(owner["uid"])
|
||||
assert (store._plot_at(farm["uid"], 0).get("crop_key") or "") == ""
|
||||
plot = store._plot_at(farm["uid"], 0)
|
||||
assert (plot.get("crop_key") or "") == "shell"
|
||||
assert 0 < float(plot.get("raided_fraction") or 0) <= 1.0
|
||||
|
||||
|
||||
def test_owner_still_harvests_the_unraided_remainder(local_db):
|
||||
owner = _reset("unit_owner")
|
||||
thief = _reset("unit_thief", coins=0)
|
||||
store.plant(owner, 0, "shell")
|
||||
_ripen_past_grace(owner)
|
||||
before = _coins(owner)
|
||||
stolen = store.steal(thief, owner, 0)["coins"]
|
||||
result = store.harvest(owner, 0)
|
||||
harvested = result["coins"]
|
||||
assert harvested > 0
|
||||
assert _coins(owner) == before + harvested
|
||||
full = economy.realizable_harvest_coins(
|
||||
economy.crop_for("shell"), golden=result["golden"]
|
||||
)
|
||||
assert harvested + stolen <= full
|
||||
|
||||
|
||||
def test_steal_blocked_once_the_build_is_fully_stripped(local_db):
|
||||
owner = _reset("unit_owner")
|
||||
store.plant(owner, 0, "shell")
|
||||
_ripen_past_grace(owner)
|
||||
farm = store.get_farm(owner["uid"])
|
||||
plot = store._plot_at(farm["uid"], 0)
|
||||
get_table("game_plots").update(
|
||||
{"uid": plot["uid"], "raided_fraction": 1.0}, ["uid"]
|
||||
)
|
||||
thief = _reset("unit_thief", coins=0)
|
||||
with pytest.raises(GameError):
|
||||
store.steal(thief, owner, 0)
|
||||
|
||||
|
||||
def test_steal_capped_per_victim_per_day(local_db):
|
||||
owner = _reset("unit_owner")
|
||||
store.plant(owner, 0, "shell")
|
||||
_ripen_past_grace(owner)
|
||||
for index in range(economy.STEAL_MAX_PER_VICTIM_PER_DAY):
|
||||
thief = _reset(f"unit_thief_{index}", coins=0)
|
||||
store.steal(thief, owner, 0)
|
||||
farm = store.get_farm(owner["uid"])
|
||||
plot = store._plot_at(farm["uid"], 0)
|
||||
get_table("game_plots").update(
|
||||
{"uid": plot["uid"], "raided_fraction": 0.0}, ["uid"]
|
||||
)
|
||||
blocked = _reset("unit_thief_last", coins=0)
|
||||
with pytest.raises(GameError):
|
||||
store.steal(blocked, owner, 0)
|
||||
|
||||
|
||||
def test_steal_protected_within_grace(local_db):
|
||||
@@ -631,6 +683,7 @@ def test_steal_cooldown_blocks_second_raid(local_db):
|
||||
store.plant(owner, 0, "shell")
|
||||
_ripen_past_grace(owner)
|
||||
store.steal(thief, owner, 0)
|
||||
store.harvest(owner, 0)
|
||||
store.plant(owner, 0, "shell")
|
||||
_ripen_past_grace(owner)
|
||||
with pytest.raises(GameError):
|
||||
|
||||
@@ -0,0 +1 @@
|
||||
# retoor <retoor@molodetz.nl>
|
||||
@@ -0,0 +1,235 @@
|
||||
# retoor <retoor@molodetz.nl>
|
||||
|
||||
from datetime import datetime, timezone
|
||||
|
||||
import pytest
|
||||
from pydantic import ValidationError
|
||||
|
||||
from devplacepy.config import QUIZ_MAX_OPTIONS, QUIZ_MAX_QUESTIONS
|
||||
from devplacepy.database import get_table
|
||||
from devplacepy.models import QuizDocument, QuizImportForm
|
||||
from devplacepy.services.quiz import store
|
||||
from devplacepy.services.quiz.store import QuizError
|
||||
from devplacepy.utils import generate_uid, make_combined_slug
|
||||
|
||||
_counter = [0]
|
||||
|
||||
DOCUMENT = {
|
||||
"title": "SQLite fundamentals",
|
||||
"description": "Questions on WAL and indexing.",
|
||||
"settings": {"shuffle_questions": True, "reveal_answers": True, "pass_percent": 70},
|
||||
"questions": [
|
||||
{
|
||||
"kind": "single_choice",
|
||||
"prompt": "Which journal mode allows concurrent readers?",
|
||||
"points": 2,
|
||||
"explanation": "WAL keeps readers off the writer's lock.",
|
||||
"options": [{"label": "DELETE"}, {"label": "WAL", "is_correct": True}],
|
||||
},
|
||||
{
|
||||
"kind": "free_text",
|
||||
"prompt": "Why does a partial index help?",
|
||||
"points": 3,
|
||||
"expected_answer": "It keeps the live query off a one bucket index.",
|
||||
"grading_criteria": "Accept any answer mentioning the planner.",
|
||||
},
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
def _user(prefix):
|
||||
_counter[0] += 1
|
||||
uid = generate_uid()
|
||||
get_table("users").insert(
|
||||
{
|
||||
"uid": uid,
|
||||
"username": f"{prefix}{_counter[0]}",
|
||||
"email": f"{prefix}{_counter[0]}@t.dev",
|
||||
"role": "Member",
|
||||
"api_key": generate_uid(),
|
||||
"created_at": datetime.now(timezone.utc).isoformat(),
|
||||
}
|
||||
)
|
||||
return get_table("users").find_one(uid=uid)
|
||||
|
||||
|
||||
def _quiz(owner, document):
|
||||
uid = generate_uid()
|
||||
get_table("quizzes").insert(
|
||||
store.born_live(
|
||||
{
|
||||
"uid": uid,
|
||||
"user_uid": owner["uid"],
|
||||
"slug": make_combined_slug(document.title, uid),
|
||||
"title": document.title,
|
||||
"description": document.description,
|
||||
"status": "draft",
|
||||
"published_at": "",
|
||||
"question_count": 0,
|
||||
"total_points": 0,
|
||||
"attempt_count": 0,
|
||||
"stars": 0,
|
||||
**store.document_settings(document),
|
||||
}
|
||||
)
|
||||
)
|
||||
store.import_questions(uid, document)
|
||||
return store.get_quiz(uid)
|
||||
|
||||
|
||||
def test_parse_document_accepts_the_reference_shape():
|
||||
document = store.parse_document(DOCUMENT)
|
||||
assert document.title == "SQLite fundamentals"
|
||||
assert len(document.questions) == 2
|
||||
|
||||
|
||||
def test_parse_document_rejects_junk():
|
||||
with pytest.raises(QuizError):
|
||||
store.parse_document({"title": "x"})
|
||||
|
||||
|
||||
def test_the_import_form_parses_a_json_string():
|
||||
import json
|
||||
|
||||
form = QuizImportForm(document=json.dumps(DOCUMENT))
|
||||
assert form.document.title == "SQLite fundamentals"
|
||||
|
||||
|
||||
def test_the_import_form_rejects_invalid_json():
|
||||
with pytest.raises(ValidationError):
|
||||
QuizImportForm(document="{not json")
|
||||
|
||||
|
||||
def test_a_document_needs_at_least_one_question():
|
||||
with pytest.raises(ValidationError):
|
||||
QuizDocument(title="Empty quiz", questions=[])
|
||||
|
||||
|
||||
def test_a_document_is_capped_at_the_question_limit():
|
||||
question = {"kind": "numeric", "prompt": "q", "numeric_value": 1}
|
||||
with pytest.raises(ValidationError):
|
||||
QuizDocument(title="Too big", questions=[question] * (QUIZ_MAX_QUESTIONS + 1))
|
||||
|
||||
|
||||
def test_a_question_is_capped_at_the_option_limit():
|
||||
options = [{"label": f"L{index}"} for index in range(QUIZ_MAX_OPTIONS + 1)]
|
||||
options[0]["is_correct"] = True
|
||||
with pytest.raises(ValidationError):
|
||||
QuizDocument(
|
||||
title="Too many options",
|
||||
questions=[{"kind": "single_choice", "prompt": "q", "options": options}],
|
||||
)
|
||||
|
||||
|
||||
def test_a_single_choice_needs_exactly_one_correct_option():
|
||||
with pytest.raises(ValidationError):
|
||||
QuizDocument(
|
||||
title="Bad choice",
|
||||
questions=[
|
||||
{
|
||||
"kind": "single_choice",
|
||||
"prompt": "q",
|
||||
"options": [{"label": "a"}, {"label": "b"}],
|
||||
}
|
||||
],
|
||||
)
|
||||
|
||||
|
||||
def test_a_free_text_needs_a_reference_or_criteria():
|
||||
with pytest.raises(ValidationError):
|
||||
QuizDocument(
|
||||
title="Bad free text",
|
||||
questions=[{"kind": "free_text", "prompt": "explain"}],
|
||||
)
|
||||
|
||||
|
||||
def test_a_matching_needs_a_right_hand_value_on_every_pair():
|
||||
with pytest.raises(ValidationError):
|
||||
QuizDocument(
|
||||
title="Bad matching",
|
||||
questions=[
|
||||
{
|
||||
"kind": "matching",
|
||||
"prompt": "q",
|
||||
"options": [{"label": "a", "match_value": "A"}, {"label": "b"}],
|
||||
}
|
||||
],
|
||||
)
|
||||
|
||||
|
||||
def test_document_settings_map_onto_the_quiz_columns():
|
||||
settings = store.document_settings(store.parse_document(DOCUMENT))
|
||||
assert settings["shuffle_questions"] == 1
|
||||
assert settings["reveal_answers"] == 1
|
||||
assert settings["pass_percent"] == 70
|
||||
|
||||
|
||||
def test_importing_creates_every_question_in_order(local_db):
|
||||
owner = _user("qd")
|
||||
quiz = _quiz(owner, store.parse_document(DOCUMENT))
|
||||
questions = store.list_questions(quiz["uid"])
|
||||
assert [q["kind"] for q in questions] == ["single_choice", "free_text"]
|
||||
assert [q["position"] for q in questions] == [0, 1]
|
||||
|
||||
|
||||
def test_importing_recomputes_the_totals(local_db):
|
||||
owner = _user("qd")
|
||||
quiz = _quiz(owner, store.parse_document(DOCUMENT))
|
||||
assert quiz["question_count"] == 2
|
||||
assert quiz["total_points"] == 5
|
||||
|
||||
|
||||
def test_importing_stores_the_kind_specific_fields(local_db):
|
||||
owner = _user("qd")
|
||||
quiz = _quiz(owner, store.parse_document(DOCUMENT))
|
||||
free_text = store.list_questions(quiz["uid"])[1]
|
||||
assert free_text["expected_answer"].startswith("It keeps")
|
||||
assert free_text["grading_criteria"].startswith("Accept")
|
||||
|
||||
|
||||
def test_the_owner_export_round_trips_through_import(local_db):
|
||||
owner = _user("qd")
|
||||
quiz = _quiz(owner, store.parse_document(DOCUMENT))
|
||||
exported = store.export_document(quiz["uid"], True)
|
||||
reimported = _quiz(owner, store.parse_document(exported))
|
||||
assert reimported["question_count"] == quiz["question_count"]
|
||||
assert reimported["total_points"] == quiz["total_points"]
|
||||
assert store.export_document(reimported["uid"], True) == exported
|
||||
|
||||
|
||||
def test_the_owner_export_carries_the_answer_key(local_db):
|
||||
owner = _user("qd")
|
||||
quiz = _quiz(owner, store.parse_document(DOCUMENT))
|
||||
exported = store.export_document(quiz["uid"], True)
|
||||
assert any(option.get("is_correct") for option in exported["questions"][0]["options"])
|
||||
assert exported["questions"][1]["expected_answer"]
|
||||
|
||||
|
||||
def test_the_public_export_omits_the_answer_key(local_db):
|
||||
owner = _user("qd")
|
||||
quiz = _quiz(owner, store.parse_document(DOCUMENT))
|
||||
exported = store.export_document(quiz["uid"], False)
|
||||
assert all("is_correct" not in option for option in exported["questions"][0]["options"])
|
||||
assert "expected_answer" not in exported["questions"][1]
|
||||
assert "correct_boolean" not in exported["questions"][0]
|
||||
|
||||
|
||||
def test_the_public_export_keeps_the_prompts_and_labels(local_db):
|
||||
owner = _user("qd")
|
||||
quiz = _quiz(owner, store.parse_document(DOCUMENT))
|
||||
exported = store.export_document(quiz["uid"], False)
|
||||
assert exported["questions"][0]["prompt"].startswith("Which journal mode")
|
||||
assert [option["label"] for option in exported["questions"][0]["options"]] == ["DELETE", "WAL"]
|
||||
|
||||
|
||||
def test_the_export_carries_the_settings(local_db):
|
||||
owner = _user("qd")
|
||||
quiz = _quiz(owner, store.parse_document(DOCUMENT))
|
||||
exported = store.export_document(quiz["uid"], False)
|
||||
assert exported["settings"]["pass_percent"] == 70
|
||||
assert exported["settings"]["reveal_answers"] is True
|
||||
|
||||
|
||||
def test_exporting_an_unknown_quiz_raises(local_db):
|
||||
with pytest.raises(QuizError):
|
||||
store.export_document("does-not-exist", True)
|
||||
@@ -0,0 +1,149 @@
|
||||
# retoor <retoor@molodetz.nl>
|
||||
|
||||
import json
|
||||
|
||||
from devplacepy.config import QUIZ_FEEDBACK_MAX_CHARS
|
||||
from devplacepy.services.quiz import grading, scoring
|
||||
|
||||
QUESTION = {
|
||||
"uid": "q1",
|
||||
"prompt": "Why does a partial index help here?",
|
||||
"expected_answer": "It keeps the live IS NULL query off a one bucket index.",
|
||||
"grading_criteria": "Accept any answer mentioning the planner.",
|
||||
"points": 3,
|
||||
}
|
||||
|
||||
|
||||
def test_parse_verdict_reads_a_plain_object():
|
||||
parsed = grading.parse_verdict('{"correct": true, "score": 0.8}')
|
||||
assert parsed == {"correct": True, "score": 0.8}
|
||||
|
||||
|
||||
def test_parse_verdict_tolerates_a_code_fence():
|
||||
parsed = grading.parse_verdict('```json\n{"score": 0.5}\n```')
|
||||
assert parsed == {"score": 0.5}
|
||||
|
||||
|
||||
def test_parse_verdict_tolerates_surrounding_prose():
|
||||
parsed = grading.parse_verdict('Here you go: {"score": 1.0} hope that helps')
|
||||
assert parsed == {"score": 1.0}
|
||||
|
||||
|
||||
def test_parse_verdict_returns_none_for_junk():
|
||||
assert grading.parse_verdict("not json at all") is None
|
||||
assert grading.parse_verdict("") is None
|
||||
|
||||
|
||||
def test_parse_verdict_returns_none_for_a_json_list():
|
||||
assert grading.parse_verdict("[1, 2, 3]") is None
|
||||
|
||||
|
||||
def test_build_result_clamps_a_high_score():
|
||||
result = grading.build_result({"score": 4.0, "correct": True})
|
||||
assert result.score == 1.0
|
||||
|
||||
|
||||
def test_build_result_clamps_a_negative_score():
|
||||
result = grading.build_result({"score": -2.0, "correct": False})
|
||||
assert result.score == 0.0
|
||||
|
||||
|
||||
def test_build_result_derives_correct_from_the_clamped_score():
|
||||
result = grading.build_result({"correct": True, "score": 0.0})
|
||||
assert result.is_correct is False
|
||||
|
||||
|
||||
def test_build_result_marks_a_high_score_correct_despite_the_model():
|
||||
result = grading.build_result({"correct": False, "score": 0.9})
|
||||
assert result.is_correct is True
|
||||
|
||||
|
||||
def test_build_result_falls_back_to_the_boolean_without_a_score():
|
||||
assert grading.build_result({"correct": True}).score == 1.0
|
||||
assert grading.build_result({"correct": False}).score == 0.0
|
||||
|
||||
|
||||
def test_build_result_handles_an_unparsable_score():
|
||||
assert grading.build_result({"correct": True, "score": "lots"}).score == 1.0
|
||||
|
||||
|
||||
def test_build_result_clamps_the_confidence():
|
||||
assert grading.build_result({"score": 1.0, "confidence": 9.0}).confidence == 1.0
|
||||
assert grading.build_result({"score": 1.0, "confidence": -9.0}).confidence == 0.0
|
||||
|
||||
|
||||
def test_build_result_truncates_and_strips_feedback():
|
||||
parsed = {"score": 1.0, "feedback": "<b>nice</b> " + "x" * (QUIZ_FEEDBACK_MAX_CHARS + 50)}
|
||||
result = grading.build_result(parsed)
|
||||
assert "<b>" not in result.feedback
|
||||
assert len(result.feedback) <= QUIZ_FEEDBACK_MAX_CHARS
|
||||
|
||||
|
||||
def test_build_result_always_has_feedback():
|
||||
assert grading.build_result({"score": 1.0}).feedback
|
||||
|
||||
|
||||
def test_build_result_is_stamped_as_ai():
|
||||
assert grading.build_result({"score": 1.0}).graded_by == "ai"
|
||||
|
||||
|
||||
def test_build_prompt_carries_the_grading_context():
|
||||
payload = json.loads(grading.build_prompt(QUESTION, "because the planner"))
|
||||
assert payload["reference_answer"] == QUESTION["expected_answer"]
|
||||
assert payload["grading_criteria"] == QUESTION["grading_criteria"]
|
||||
assert payload["learner_answer"] == "because the planner"
|
||||
|
||||
|
||||
def test_system_prompt_treats_the_answer_as_data():
|
||||
assert "DATA" in grading.SYSTEM_PROMPT
|
||||
assert "JSON" in grading.SYSTEM_PROMPT
|
||||
|
||||
|
||||
def test_grade_free_text_falls_back_without_an_api_key():
|
||||
result = grading.grade_free_text("", QUESTION, "the planner picks a bad index")
|
||||
assert result.graded_by == "fallback"
|
||||
assert 0.0 <= result.score <= 1.0
|
||||
|
||||
|
||||
def test_grade_free_text_falls_back_when_the_gateway_raises(monkeypatch):
|
||||
def boom(*args, **kwargs):
|
||||
raise RuntimeError("gateway down")
|
||||
|
||||
monkeypatch.setattr(grading, "gateway_complete", boom)
|
||||
result = grading.grade_free_text("key", QUESTION, "the planner picks a bad index")
|
||||
assert result.graded_by == "fallback"
|
||||
assert "unavailable" in result.feedback
|
||||
|
||||
|
||||
def test_grade_free_text_falls_back_on_an_unreadable_reply(monkeypatch):
|
||||
monkeypatch.setattr(grading, "gateway_complete", lambda *a, **k: ("not json", None))
|
||||
result = grading.grade_free_text("key", QUESTION, "the planner picks a bad index")
|
||||
assert result.graded_by == "fallback"
|
||||
|
||||
|
||||
def test_grade_free_text_uses_a_valid_verdict(monkeypatch):
|
||||
reply = json.dumps({"correct": True, "score": 0.75, "feedback": "Close.", "confidence": 0.9})
|
||||
monkeypatch.setattr(grading, "gateway_complete", lambda *a, **k: (reply, None))
|
||||
result = grading.grade_free_text("key", QUESTION, "the planner")
|
||||
assert result.graded_by == "ai"
|
||||
assert result.score == 0.75
|
||||
assert result.is_correct is True
|
||||
assert result.feedback == "Close."
|
||||
|
||||
|
||||
def test_grade_free_text_passes_the_answering_key_through(monkeypatch):
|
||||
seen = {}
|
||||
|
||||
def capture(api_key, system, text, timeout):
|
||||
seen["api_key"] = api_key
|
||||
return json.dumps({"score": 1.0}), None
|
||||
|
||||
monkeypatch.setattr(grading, "gateway_complete", capture)
|
||||
grading.grade_free_text("member-key", QUESTION, "answer")
|
||||
assert seen["api_key"] == "member-key"
|
||||
|
||||
|
||||
def test_the_fallback_score_is_the_deterministic_overlap():
|
||||
expected = scoring.token_overlap_score(QUESTION["expected_answer"], "one bucket index")
|
||||
result = grading.grade_free_text("", QUESTION, "one bucket index")
|
||||
assert result.score == expected
|
||||
@@ -0,0 +1,387 @@
|
||||
# retoor <retoor@molodetz.nl>
|
||||
|
||||
from datetime import datetime, timezone
|
||||
|
||||
from devplacepy.database import get_table
|
||||
from devplacepy.services.quiz import store
|
||||
from devplacepy.utils import generate_uid, make_combined_slug
|
||||
|
||||
_counter = [0]
|
||||
|
||||
|
||||
def _user(prefix):
|
||||
_counter[0] += 1
|
||||
uid = generate_uid()
|
||||
get_table("users").insert(
|
||||
{
|
||||
"uid": uid,
|
||||
"username": f"{prefix}{_counter[0]}",
|
||||
"email": f"{prefix}{_counter[0]}@t.dev",
|
||||
"role": "Member",
|
||||
"api_key": generate_uid(),
|
||||
"created_at": datetime.now(timezone.utc).isoformat(),
|
||||
}
|
||||
)
|
||||
return get_table("users").find_one(uid=uid)
|
||||
|
||||
|
||||
def _published(owner, points=10):
|
||||
uid = generate_uid()
|
||||
get_table("quizzes").insert(
|
||||
store.born_live(
|
||||
{
|
||||
"uid": uid,
|
||||
"user_uid": owner["uid"],
|
||||
"slug": make_combined_slug("Board", uid),
|
||||
"title": "Board",
|
||||
"description": "",
|
||||
"status": "published",
|
||||
"published_at": datetime.now(timezone.utc).isoformat(),
|
||||
"shuffle_questions": 0,
|
||||
"shuffle_options": 0,
|
||||
"reveal_answers": 0,
|
||||
"allow_review": 1,
|
||||
"time_limit_seconds": 0,
|
||||
"pass_percent": 0,
|
||||
"question_count": 1,
|
||||
"total_points": points,
|
||||
"attempt_count": 0,
|
||||
"stars": 0,
|
||||
}
|
||||
)
|
||||
)
|
||||
return store.get_quiz(uid)
|
||||
|
||||
|
||||
def _completed(player, quiz, points, percent):
|
||||
uid = generate_uid()
|
||||
get_table("quiz_attempts").insert(
|
||||
store.born_live(
|
||||
{
|
||||
"uid": uid,
|
||||
"quiz_uid": quiz["uid"],
|
||||
"user_uid": player["uid"],
|
||||
"status": "completed",
|
||||
"question_order": "[]",
|
||||
"started_at": datetime.now(timezone.utc).isoformat(),
|
||||
"expires_at": "",
|
||||
"completed_at": datetime.now(timezone.utc).isoformat(),
|
||||
"answered_count": 1,
|
||||
"score_points": float(points),
|
||||
"max_points": int(quiz["total_points"]),
|
||||
"score_percent": float(percent),
|
||||
"passed": 0,
|
||||
}
|
||||
)
|
||||
)
|
||||
store.clear_cache()
|
||||
return uid
|
||||
|
||||
|
||||
def _in_progress(player, quiz):
|
||||
uid = generate_uid()
|
||||
get_table("quiz_attempts").insert(
|
||||
store.born_live(
|
||||
{
|
||||
"uid": uid,
|
||||
"quiz_uid": quiz["uid"],
|
||||
"user_uid": player["uid"],
|
||||
"status": "in_progress",
|
||||
"question_order": "[]",
|
||||
"started_at": datetime.now(timezone.utc).isoformat(),
|
||||
"expires_at": "",
|
||||
"completed_at": "",
|
||||
"answered_count": 0,
|
||||
"score_points": 0.0,
|
||||
"max_points": int(quiz["total_points"]),
|
||||
"score_percent": 0.0,
|
||||
"passed": 0,
|
||||
}
|
||||
)
|
||||
)
|
||||
store.clear_cache()
|
||||
return uid
|
||||
|
||||
|
||||
def _entry_for(player):
|
||||
for entry in store.scoreboard(100):
|
||||
if entry["user"]["uid"] == player["uid"]:
|
||||
return entry
|
||||
return None
|
||||
|
||||
|
||||
def test_a_completed_attempt_puts_a_member_on_the_board(local_db):
|
||||
owner = _user("sb")
|
||||
player = _user("sb")
|
||||
quiz = _published(owner)
|
||||
_completed(player, quiz, 8, 80.0)
|
||||
entry = _entry_for(player)
|
||||
assert entry["total_points"] == 8.0
|
||||
assert entry["quizzes_completed"] == 1
|
||||
|
||||
|
||||
def test_only_the_best_attempt_per_quiz_counts(local_db):
|
||||
owner = _user("sb")
|
||||
player = _user("sb")
|
||||
quiz = _published(owner)
|
||||
_completed(player, quiz, 4, 40.0)
|
||||
_completed(player, quiz, 9, 90.0)
|
||||
_completed(player, quiz, 2, 20.0)
|
||||
entry = _entry_for(player)
|
||||
assert entry["total_points"] == 9.0
|
||||
assert entry["quizzes_completed"] == 1
|
||||
|
||||
|
||||
def test_replaying_a_quiz_can_never_multiply_a_score(local_db):
|
||||
owner = _user("sb")
|
||||
player = _user("sb")
|
||||
quiz = _published(owner)
|
||||
_completed(player, quiz, 7, 70.0)
|
||||
before = _entry_for(player)["total_points"]
|
||||
for _ in range(5):
|
||||
_completed(player, quiz, 7, 70.0)
|
||||
assert _entry_for(player)["total_points"] == before
|
||||
|
||||
|
||||
def test_a_worse_attempt_never_lowers_the_contribution(local_db):
|
||||
owner = _user("sb")
|
||||
player = _user("sb")
|
||||
quiz = _published(owner)
|
||||
_completed(player, quiz, 9, 90.0)
|
||||
_completed(player, quiz, 1, 10.0)
|
||||
assert _entry_for(player)["total_points"] == 9.0
|
||||
|
||||
|
||||
def test_a_better_attempt_raises_by_exactly_the_difference(local_db):
|
||||
owner = _user("sb")
|
||||
player = _user("sb")
|
||||
quiz = _published(owner)
|
||||
_completed(player, quiz, 4, 40.0)
|
||||
_completed(player, quiz, 10, 100.0)
|
||||
assert _entry_for(player)["total_points"] == 10.0
|
||||
|
||||
|
||||
def test_two_quizzes_sum_their_best_attempts(local_db):
|
||||
owner = _user("sb")
|
||||
player = _user("sb")
|
||||
first = _published(owner)
|
||||
second = _published(owner)
|
||||
_completed(player, first, 6, 60.0)
|
||||
_completed(player, second, 3, 30.0)
|
||||
entry = _entry_for(player)
|
||||
assert entry["total_points"] == 9.0
|
||||
assert entry["quizzes_completed"] == 2
|
||||
|
||||
|
||||
def test_an_author_scores_on_their_own_quiz(local_db):
|
||||
owner = _user("sb")
|
||||
quiz = _published(owner)
|
||||
_completed(owner, quiz, 10, 100.0)
|
||||
entry = _entry_for(owner)
|
||||
assert entry["total_points"] == 10.0
|
||||
assert entry["quizzes_completed"] == 1
|
||||
assert entry["perfect_count"] == 1
|
||||
|
||||
|
||||
def test_an_in_progress_attempt_is_not_counted(local_db):
|
||||
owner = _user("sb")
|
||||
player = _user("sb")
|
||||
quiz = _published(owner)
|
||||
_in_progress(player, quiz)
|
||||
assert _entry_for(player) is None
|
||||
|
||||
|
||||
def test_a_draft_quiz_is_not_counted(local_db):
|
||||
owner = _user("sb")
|
||||
player = _user("sb")
|
||||
quiz = _published(owner)
|
||||
get_table("quizzes").update({"uid": quiz["uid"], "status": "draft"}, ["uid"])
|
||||
_completed(player, quiz, 10, 100.0)
|
||||
assert _entry_for(player) is None
|
||||
|
||||
|
||||
def test_a_deleted_attempt_is_not_counted(local_db):
|
||||
owner = _user("sb")
|
||||
player = _user("sb")
|
||||
quiz = _published(owner)
|
||||
attempt_uid = _completed(player, quiz, 10, 100.0)
|
||||
get_table("quiz_attempts").update(
|
||||
{"uid": attempt_uid, "deleted_at": datetime.now(timezone.utc).isoformat()}, ["uid"]
|
||||
)
|
||||
store.clear_cache()
|
||||
assert _entry_for(player) is None
|
||||
|
||||
|
||||
def test_ranks_are_a_gapless_total_order(local_db):
|
||||
owner = _user("sb")
|
||||
quiz = _published(owner)
|
||||
for score in (3, 9, 6):
|
||||
_completed(_user("sb"), quiz, score, score * 10.0)
|
||||
ranks = [entry["rank"] for entry in store.scoreboard(100)]
|
||||
assert ranks == list(range(1, len(ranks) + 1))
|
||||
|
||||
|
||||
def test_equal_totals_resolve_deterministically(local_db):
|
||||
owner = _user("sb")
|
||||
quiz = _published(owner)
|
||||
for _ in range(3):
|
||||
_completed(_user("sb"), quiz, 5, 50.0)
|
||||
first = [entry["user"]["uid"] for entry in store.scoreboard(100)]
|
||||
store.clear_cache()
|
||||
assert [entry["user"]["uid"] for entry in store.scoreboard(100)] == first
|
||||
|
||||
|
||||
def test_the_limit_bounds_the_board(local_db):
|
||||
owner = _user("sb")
|
||||
quiz = _published(owner)
|
||||
for score in range(1, 6):
|
||||
_completed(_user("sb"), quiz, score, score * 10.0)
|
||||
assert len(store.scoreboard(3)) == 3
|
||||
|
||||
|
||||
def test_perfect_attempts_are_counted(local_db):
|
||||
owner = _user("sb")
|
||||
player = _user("sb")
|
||||
quiz = _published(owner)
|
||||
_completed(player, quiz, 10, 100.0)
|
||||
assert _entry_for(player)["perfect_count"] == 1
|
||||
|
||||
|
||||
def test_standing_for_finds_a_member_outside_the_top(local_db):
|
||||
owner = _user("sb")
|
||||
player = _user("sb")
|
||||
quiz = _published(owner)
|
||||
_completed(player, quiz, 2, 20.0)
|
||||
standing = store.standing_for(player["uid"])
|
||||
assert standing["user"]["uid"] == player["uid"]
|
||||
assert standing["rank"] >= 1
|
||||
|
||||
|
||||
def test_standing_for_is_none_without_attempts(local_db):
|
||||
assert store.standing_for(_user("sb")["uid"]) is None
|
||||
|
||||
|
||||
def test_attempt_states_reports_done_with_the_best_percentage(local_db):
|
||||
owner = _user("sb")
|
||||
player = _user("sb")
|
||||
quiz = _published(owner)
|
||||
_completed(player, quiz, 4, 40.0)
|
||||
_completed(player, quiz, 9, 90.0)
|
||||
states = store.attempt_states_for(player["uid"], [quiz["uid"]])
|
||||
assert states[quiz["uid"]]["state"] == "done"
|
||||
assert states[quiz["uid"]]["best_percent"] == 90.0
|
||||
|
||||
|
||||
def test_attempt_states_reports_in_progress(local_db):
|
||||
owner = _user("sb")
|
||||
player = _user("sb")
|
||||
quiz = _published(owner)
|
||||
_in_progress(player, quiz)
|
||||
states = store.attempt_states_for(player["uid"], [quiz["uid"]])
|
||||
assert states[quiz["uid"]]["state"] == "in_progress"
|
||||
assert states[quiz["uid"]]["attempt_uid"]
|
||||
|
||||
|
||||
def test_attempt_states_is_empty_for_a_guest(local_db):
|
||||
owner = _user("sb")
|
||||
quiz = _published(owner)
|
||||
assert store.attempt_states_for("", [quiz["uid"]]) == {}
|
||||
|
||||
|
||||
def test_attempt_states_is_empty_without_quiz_uids(local_db):
|
||||
assert store.attempt_states_for(_user("sb")["uid"], []) == {}
|
||||
|
||||
|
||||
def test_completed_quiz_uids_lists_only_finished_quizzes(local_db):
|
||||
owner = _user("sb")
|
||||
player = _user("sb")
|
||||
done = _published(owner)
|
||||
open_quiz = _published(owner)
|
||||
_completed(player, done, 5, 50.0)
|
||||
_in_progress(player, open_quiz)
|
||||
completed = store.completed_quiz_uids(player["uid"])
|
||||
assert done["uid"] in completed
|
||||
assert open_quiz["uid"] not in completed
|
||||
|
||||
|
||||
def test_progress_for_summarises_the_viewer(local_db):
|
||||
owner = _user("sb")
|
||||
player = _user("sb")
|
||||
quiz = _published(owner)
|
||||
_completed(player, quiz, 10, 100.0)
|
||||
progress = store.progress_for(player["uid"])
|
||||
assert progress["completed"] == 1
|
||||
assert progress["avg_percent"] == 100.0
|
||||
assert progress["perfect_count"] == 1
|
||||
|
||||
|
||||
def test_the_quiz_leaderboard_ranks_by_percentage(local_db):
|
||||
owner = _user("sb")
|
||||
quiz = _published(owner)
|
||||
_completed(_user("sb"), quiz, 3, 30.0)
|
||||
_completed(_user("sb"), quiz, 9, 90.0)
|
||||
entries = store.quiz_leaderboard(quiz["uid"])
|
||||
assert entries[0]["score_percent"] == 90.0
|
||||
assert entries[0]["rank"] == 1
|
||||
|
||||
|
||||
def test_the_quiz_leaderboard_ignores_unfinished_attempts(local_db):
|
||||
owner = _user("sb")
|
||||
quiz = _published(owner)
|
||||
_in_progress(_user("sb"), quiz)
|
||||
assert store.quiz_leaderboard(quiz["uid"]) == []
|
||||
|
||||
|
||||
def test_progress_agrees_with_the_scoreboard_for_an_author_only_completion(local_db):
|
||||
author = _user("sb")
|
||||
quiz = _published(author)
|
||||
_completed(author, quiz, 10, 100.0)
|
||||
progress = store.progress_for(author["uid"])
|
||||
entry = _entry_for(author)
|
||||
assert progress["completed"] == entry["quizzes_completed"] == 1
|
||||
assert progress["avg_percent"] == entry["avg_percent"] == 100.0
|
||||
assert progress["total_points"] == entry["total_points"] == 10.0
|
||||
assert progress["perfect_count"] == entry["perfect_count"] == 1
|
||||
assert progress["rank"] == entry["rank"]
|
||||
|
||||
|
||||
def test_progress_counts_a_ranked_completion(local_db):
|
||||
owner = _user("sb")
|
||||
player = _user("sb")
|
||||
quiz = _published(owner)
|
||||
_completed(player, quiz, 8, 80.0)
|
||||
progress = store.progress_for(player["uid"])
|
||||
entry = _entry_for(player)
|
||||
assert progress["completed"] == entry["quizzes_completed"] == 1
|
||||
assert progress["avg_percent"] == entry["avg_percent"] == 80.0
|
||||
assert progress["total_points"] == entry["total_points"] == 8.0
|
||||
|
||||
|
||||
def test_completing_your_own_quiz_clears_it_from_todo(local_db):
|
||||
author = _user("sb")
|
||||
before = store.progress_for(author["uid"])["todo"]
|
||||
quiz = _published(author)
|
||||
assert store.progress_for(author["uid"])["todo"] == before + 1
|
||||
_completed(author, quiz, 10, 100.0)
|
||||
assert store.progress_for(author["uid"])["todo"] == before
|
||||
|
||||
|
||||
def test_progress_todo_counts_every_published_quiz(local_db):
|
||||
author = _user("sb")
|
||||
other = _user("sb")
|
||||
before = store.progress_for(author["uid"])["todo"]
|
||||
_published(author)
|
||||
_published(other)
|
||||
_published(other)
|
||||
assert store.progress_for(author["uid"])["todo"] == before + 3
|
||||
|
||||
|
||||
def test_completing_a_quiz_moves_it_from_todo_to_completed(local_db):
|
||||
owner = _user("sb")
|
||||
player = _user("sb")
|
||||
quiz = _published(owner)
|
||||
before = store.progress_for(player["uid"])
|
||||
_completed(player, quiz, 5, 50.0)
|
||||
after = store.progress_for(player["uid"])
|
||||
assert after["completed"] == before["completed"] + 1
|
||||
assert after["todo"] == before["todo"] - 1
|
||||
@@ -0,0 +1,278 @@
|
||||
# retoor <retoor@molodetz.nl>
|
||||
|
||||
import json
|
||||
from datetime import datetime, timedelta, timezone
|
||||
|
||||
import pytest
|
||||
|
||||
from devplacepy.services.quiz import scoring
|
||||
|
||||
OPTIONS = [
|
||||
{"uid": "o0", "position": 0, "label": "L0", "match_value": "M0", "is_correct": 1},
|
||||
{"uid": "o1", "position": 1, "label": "L1", "match_value": "M1", "is_correct": 1},
|
||||
{"uid": "o2", "position": 2, "label": "L2", "match_value": "M2", "is_correct": 0},
|
||||
{"uid": "o3", "position": 3, "label": "L3", "match_value": "M3", "is_correct": 0},
|
||||
]
|
||||
|
||||
NOW = datetime(2026, 7, 25, 12, 0, 0, tzinfo=timezone.utc)
|
||||
|
||||
|
||||
def question(kind, **extra):
|
||||
base = {
|
||||
"uid": "q1",
|
||||
"kind": kind,
|
||||
"points": 3,
|
||||
"correct_boolean": 0,
|
||||
"numeric_value": 0.0,
|
||||
"numeric_tolerance": 0.0,
|
||||
"case_sensitive": 0,
|
||||
}
|
||||
base.update(extra)
|
||||
return base
|
||||
|
||||
|
||||
def test_question_kinds_are_the_eight_documented_keys():
|
||||
assert scoring.KIND_KEYS == (
|
||||
"single_choice",
|
||||
"multiple_choice",
|
||||
"true_false",
|
||||
"free_text",
|
||||
"fill_blank",
|
||||
"numeric",
|
||||
"ordering",
|
||||
"matching",
|
||||
)
|
||||
|
||||
|
||||
def test_only_free_text_is_ai_graded():
|
||||
ai = [kind.key for kind in scoring.QUESTION_KINDS if kind.graded_by == "ai"]
|
||||
assert ai == ["free_text"]
|
||||
|
||||
|
||||
def test_awarded_points_is_monotonic_and_bounded():
|
||||
for points in (1, 7, 100):
|
||||
previous = -1.0
|
||||
for step in range(0, 201):
|
||||
awarded = scoring.awarded_points(points, step / 200.0)
|
||||
assert 0.0 <= awarded <= points
|
||||
assert awarded >= previous
|
||||
previous = awarded
|
||||
|
||||
|
||||
def test_awarded_points_clamps_outside_the_unit_interval():
|
||||
assert scoring.awarded_points(5, 4.0) == 5
|
||||
assert scoring.awarded_points(5, -4.0) == 0.0
|
||||
|
||||
|
||||
def test_score_percent_stays_within_bounds():
|
||||
for max_points in (0, 1, 14, 199):
|
||||
for score_points in (0, 1, 14, 400):
|
||||
assert 0.0 <= scoring.score_percent(score_points, max_points) <= 100.0
|
||||
|
||||
|
||||
def test_score_percent_is_zero_without_max_points():
|
||||
assert scoring.score_percent(10, 0) == 0.0
|
||||
|
||||
|
||||
def test_score_percent_is_exactly_full_at_max():
|
||||
assert scoring.score_percent(14, 14) == 100.0
|
||||
|
||||
|
||||
def test_passed_needs_a_pass_mark():
|
||||
assert scoring.passed(100.0, 0) is False
|
||||
assert scoring.passed(70.0, 70) is True
|
||||
assert scoring.passed(69.9, 70) is False
|
||||
|
||||
|
||||
def test_single_choice_grades_the_correct_option():
|
||||
result = scoring.grade_answer(question("single_choice"), OPTIONS, {"option_uids": ["o0"]})
|
||||
assert result.score == 1.0 and result.is_correct and result.graded_by == "auto"
|
||||
|
||||
|
||||
def test_single_choice_rejects_more_than_one_pick():
|
||||
result = scoring.grade_answer(question("single_choice"), OPTIONS, {"option_uids": ["o0", "o1"]})
|
||||
assert result.score == 0.0
|
||||
|
||||
|
||||
def test_single_choice_rejects_an_unknown_option():
|
||||
result = scoring.grade_answer(question("single_choice"), OPTIONS, {"option_uids": ["nope"]})
|
||||
assert result.score == 0.0
|
||||
|
||||
|
||||
def test_multiple_choice_gives_partial_credit():
|
||||
result = scoring.grade_answer(question("multiple_choice"), OPTIONS, {"option_uids": ["o0"]})
|
||||
assert result.score == 0.5 and not result.is_correct
|
||||
|
||||
|
||||
def test_multiple_choice_penalises_wrong_picks_but_never_below_zero():
|
||||
result = scoring.grade_answer(
|
||||
question("multiple_choice"), OPTIONS, {"option_uids": ["o2", "o3"]}
|
||||
)
|
||||
assert result.score == 0.0
|
||||
|
||||
|
||||
def test_multiple_choice_is_correct_only_on_the_exact_set():
|
||||
result = scoring.grade_answer(
|
||||
question("multiple_choice"), OPTIONS, {"option_uids": ["o0", "o1"]}
|
||||
)
|
||||
assert result.score == 1.0 and result.is_correct
|
||||
|
||||
|
||||
def test_multiple_choice_ignores_duplicate_picks():
|
||||
result = scoring.grade_answer(
|
||||
question("multiple_choice"), OPTIONS, {"option_uids": ["o0", "o0", "o1"]}
|
||||
)
|
||||
assert result.score == 1.0
|
||||
|
||||
|
||||
def test_true_false_matches_the_stored_boolean():
|
||||
prompt = question("true_false", correct_boolean=1)
|
||||
assert scoring.grade_answer(prompt, [], {"answer_text": "true"}).score == 1.0
|
||||
assert scoring.grade_answer(prompt, [], {"answer_text": "false"}).score == 0.0
|
||||
|
||||
|
||||
def test_true_false_without_an_answer_scores_zero():
|
||||
assert scoring.grade_answer(question("true_false"), [], {"answer_text": ""}).score == 0.0
|
||||
|
||||
|
||||
def test_free_text_raises_for_the_ai_grader():
|
||||
with pytest.raises(scoring.NeedsAiGrading):
|
||||
scoring.grade_answer(question("free_text"), [], {"answer_text": "anything"})
|
||||
|
||||
|
||||
def test_numeric_accepts_within_tolerance():
|
||||
prompt = question("numeric", numeric_value=10.0, numeric_tolerance=0.5)
|
||||
assert scoring.grade_answer(prompt, [], {"answer_text": "10.4"}).score == 1.0
|
||||
assert scoring.grade_answer(prompt, [], {"answer_text": "10.6"}).score == 0.0
|
||||
|
||||
|
||||
def test_numeric_accepts_a_comma_decimal_separator():
|
||||
prompt = question("numeric", numeric_value=1.5, numeric_tolerance=0.0)
|
||||
assert scoring.grade_answer(prompt, [], {"answer_text": "1,5"}).score == 1.0
|
||||
|
||||
|
||||
def test_numeric_rejects_a_non_number():
|
||||
prompt = question("numeric", numeric_value=1.0)
|
||||
assert scoring.grade_answer(prompt, [], {"answer_text": "ten"}).score == 0.0
|
||||
|
||||
|
||||
def test_fill_blank_scores_per_blank():
|
||||
prompt = question("fill_blank")
|
||||
payload = json.dumps(["M0", "wrong", "M2", "M3"])
|
||||
result = scoring.grade_answer(prompt, OPTIONS, {"answer_text": payload})
|
||||
assert result.score == 0.75
|
||||
|
||||
|
||||
def test_fill_blank_normalizes_whitespace_and_case_by_default():
|
||||
prompt = question("fill_blank")
|
||||
payload = json.dumps([" m0 ", "M1", "M2", "M3"])
|
||||
assert scoring.grade_answer(prompt, OPTIONS, {"answer_text": payload}).score == 1.0
|
||||
|
||||
|
||||
def test_fill_blank_honours_case_sensitivity():
|
||||
prompt = question("fill_blank", case_sensitive=1)
|
||||
payload = json.dumps(["m0", "M1", "M2", "M3"])
|
||||
assert scoring.grade_answer(prompt, OPTIONS, {"answer_text": payload}).score == 0.75
|
||||
|
||||
|
||||
def test_ordering_scores_the_longest_correct_prefix():
|
||||
prompt = question("ordering")
|
||||
order = ["o0", "o1", "o3", "o2"]
|
||||
assert scoring.grade_answer(prompt, OPTIONS, {"option_uids": order}).score == 0.5
|
||||
|
||||
|
||||
def test_ordering_is_correct_on_the_exact_sequence():
|
||||
prompt = question("ordering")
|
||||
order = ["o0", "o1", "o2", "o3"]
|
||||
result = scoring.grade_answer(prompt, OPTIONS, {"option_uids": order})
|
||||
assert result.score == 1.0 and result.is_correct
|
||||
|
||||
|
||||
def test_matching_scores_per_pair():
|
||||
prompt = question("matching")
|
||||
payload = json.dumps({"o0": "M0", "o1": "M1", "o2": "wrong", "o3": "wrong"})
|
||||
assert scoring.grade_answer(prompt, OPTIONS, {"answer_text": payload}).score == 0.5
|
||||
|
||||
|
||||
def test_every_kind_survives_a_malformed_submission():
|
||||
malformed = [{}, {"answer_text": None, "option_uids": None}, {"answer_text": "junk"}]
|
||||
for kind in scoring.KIND_KEYS:
|
||||
if kind == "free_text":
|
||||
continue
|
||||
for submission in malformed:
|
||||
result = scoring.grade_answer(question(kind), OPTIONS, submission)
|
||||
assert 0.0 <= result.score <= 1.0
|
||||
|
||||
|
||||
def test_an_unknown_kind_scores_zero_instead_of_raising():
|
||||
assert scoring.grade_answer(question("telepathy"), OPTIONS, {}).score == 0.0
|
||||
|
||||
|
||||
def test_question_order_is_a_stable_permutation():
|
||||
uids = [f"u{index}" for index in range(9)]
|
||||
order = scoring.question_order(uids, True, "attempt-1")
|
||||
assert sorted(order) == sorted(uids)
|
||||
assert order == scoring.question_order(uids, True, "attempt-1")
|
||||
|
||||
|
||||
def test_question_order_differs_per_seed():
|
||||
uids = [f"u{index}" for index in range(12)]
|
||||
orders = {tuple(scoring.question_order(uids, True, f"a{index}")) for index in range(40)}
|
||||
assert len(orders) > 1
|
||||
|
||||
|
||||
def test_question_order_is_untouched_without_shuffle():
|
||||
uids = ["a", "b", "c"]
|
||||
assert scoring.question_order(uids, False, "seed") == uids
|
||||
|
||||
|
||||
def test_option_order_is_a_stable_permutation():
|
||||
uids = ["o0", "o1", "o2", "o3"]
|
||||
order = scoring.option_order(uids, True, "answer-1")
|
||||
assert sorted(order) == sorted(uids)
|
||||
assert order == scoring.option_order(uids, True, "answer-1")
|
||||
|
||||
|
||||
def test_token_overlap_scores_a_perfect_match():
|
||||
assert scoring.token_overlap_score("write ahead logging", "write ahead logging") == 1.0
|
||||
|
||||
|
||||
def test_token_overlap_scores_an_empty_answer_zero():
|
||||
assert scoring.token_overlap_score("write ahead logging", "") == 0.0
|
||||
|
||||
|
||||
def test_token_overlap_stays_within_bounds():
|
||||
score = scoring.token_overlap_score("alpha beta gamma", "alpha delta")
|
||||
assert 0.0 < score < 1.0
|
||||
|
||||
|
||||
def test_attempt_expires_at_is_empty_without_a_limit():
|
||||
assert scoring.attempt_expires_at(NOW.isoformat(), 0) == ""
|
||||
|
||||
|
||||
def test_attempt_expires_at_adds_the_limit():
|
||||
expires = scoring.attempt_expires_at(NOW.isoformat(), 900)
|
||||
assert expires == (NOW + timedelta(seconds=900)).isoformat()
|
||||
|
||||
|
||||
def test_is_expired_reads_the_stored_deadline():
|
||||
expires = scoring.attempt_expires_at(NOW.isoformat(), 60)
|
||||
assert scoring.is_expired(expires, NOW) is False
|
||||
assert scoring.is_expired(expires, NOW + timedelta(seconds=61)) is True
|
||||
|
||||
|
||||
def test_is_expired_is_false_without_a_deadline():
|
||||
assert scoring.is_expired("", NOW) is False
|
||||
|
||||
|
||||
def test_remaining_seconds_never_goes_negative():
|
||||
expires = scoring.attempt_expires_at(NOW.isoformat(), 60)
|
||||
assert scoring.remaining_seconds(expires, NOW) == 60
|
||||
assert scoring.remaining_seconds(expires, NOW + timedelta(seconds=600)) == 0
|
||||
|
||||
|
||||
def test_fallback_result_is_stamped_and_explains_itself():
|
||||
result = scoring.fallback_result("write ahead logging", "write ahead logging", "gateway down")
|
||||
assert result.graded_by == "fallback"
|
||||
assert result.confidence == 0.0
|
||||
assert "unavailable" in result.feedback
|
||||
@@ -0,0 +1,531 @@
|
||||
# retoor <retoor@molodetz.nl>
|
||||
|
||||
import json
|
||||
from datetime import datetime, timedelta, timezone
|
||||
|
||||
import pytest
|
||||
|
||||
from devplacepy.database import get_table
|
||||
from devplacepy.services.quiz import store
|
||||
from devplacepy.services.quiz.store import QuizError
|
||||
from devplacepy.utils import generate_uid, make_combined_slug
|
||||
from tests.conftest import run_async
|
||||
|
||||
_counter = [0]
|
||||
|
||||
|
||||
def _user(prefix):
|
||||
_counter[0] += 1
|
||||
uid = generate_uid()
|
||||
get_table("users").insert(
|
||||
{
|
||||
"uid": uid,
|
||||
"username": f"{prefix}{_counter[0]}",
|
||||
"email": f"{prefix}{_counter[0]}@t.dev",
|
||||
"role": "Member",
|
||||
"api_key": generate_uid(),
|
||||
"created_at": datetime.now(timezone.utc).isoformat(),
|
||||
}
|
||||
)
|
||||
return get_table("users").find_one(uid=uid)
|
||||
|
||||
|
||||
def _quiz(owner, **overrides):
|
||||
uid = generate_uid()
|
||||
fields = {
|
||||
"uid": uid,
|
||||
"user_uid": owner["uid"],
|
||||
"slug": make_combined_slug("Quiz", uid),
|
||||
"title": "Quiz",
|
||||
"description": "d",
|
||||
"status": "draft",
|
||||
"published_at": "",
|
||||
"shuffle_questions": 0,
|
||||
"shuffle_options": 0,
|
||||
"reveal_answers": 1,
|
||||
"allow_review": 1,
|
||||
"time_limit_seconds": 0,
|
||||
"pass_percent": 0,
|
||||
"question_count": 0,
|
||||
"total_points": 0,
|
||||
"attempt_count": 0,
|
||||
"stars": 0,
|
||||
}
|
||||
fields.update(overrides)
|
||||
get_table("quizzes").insert(store.born_live(fields))
|
||||
return store.get_quiz(uid)
|
||||
|
||||
|
||||
def _numeric(quiz_uid, value=1.0, points=1):
|
||||
return store.add_question(
|
||||
quiz_uid,
|
||||
{
|
||||
"kind": "numeric",
|
||||
"prompt": f"Value {value}",
|
||||
"points": points,
|
||||
"numeric_value": value,
|
||||
"numeric_tolerance": 0.0,
|
||||
"options": [],
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
def _choice(quiz_uid, points=2):
|
||||
return store.add_question(
|
||||
quiz_uid,
|
||||
{
|
||||
"kind": "single_choice",
|
||||
"prompt": "Pick one",
|
||||
"points": points,
|
||||
"options": [
|
||||
{"label": "wrong", "match_value": "", "is_correct": False},
|
||||
{"label": "right", "match_value": "", "is_correct": True},
|
||||
],
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
def test_recompute_quiz_totals_reflects_the_live_questions(local_db):
|
||||
owner = _user("qs")
|
||||
quiz = _quiz(owner)
|
||||
_numeric(quiz["uid"], points=3)
|
||||
_choice(quiz["uid"], points=2)
|
||||
refreshed = store.get_quiz(quiz["uid"])
|
||||
assert refreshed["question_count"] == 2
|
||||
assert refreshed["total_points"] == 5
|
||||
|
||||
|
||||
def test_deleting_a_question_recomputes_and_renumbers(local_db):
|
||||
owner = _user("qs")
|
||||
quiz = _quiz(owner)
|
||||
first = _numeric(quiz["uid"], points=3)
|
||||
_numeric(quiz["uid"], value=2.0, points=4)
|
||||
store.delete_question(quiz["uid"], first["uid"], owner["uid"])
|
||||
refreshed = store.get_quiz(quiz["uid"])
|
||||
assert refreshed["question_count"] == 1
|
||||
assert refreshed["total_points"] == 4
|
||||
assert store.list_questions(quiz["uid"])[0]["position"] == 0
|
||||
|
||||
|
||||
def test_reorder_requires_every_question_exactly_once(local_db):
|
||||
owner = _user("qs")
|
||||
quiz = _quiz(owner)
|
||||
first = _numeric(quiz["uid"])
|
||||
_numeric(quiz["uid"], value=2.0)
|
||||
with pytest.raises(QuizError):
|
||||
store.reorder_questions(quiz["uid"], [first["uid"]])
|
||||
|
||||
|
||||
def test_reorder_applies_the_new_order(local_db):
|
||||
owner = _user("qs")
|
||||
quiz = _quiz(owner)
|
||||
first = _numeric(quiz["uid"])
|
||||
second = _numeric(quiz["uid"], value=2.0)
|
||||
store.reorder_questions(quiz["uid"], [second["uid"], first["uid"]])
|
||||
assert [q["uid"] for q in store.list_questions(quiz["uid"])] == [second["uid"], first["uid"]]
|
||||
|
||||
|
||||
def test_validation_errors_flags_an_empty_quiz(local_db):
|
||||
owner = _user("qs")
|
||||
quiz = _quiz(owner)
|
||||
assert store.validation_errors(quiz["uid"])
|
||||
|
||||
|
||||
def test_validation_errors_is_empty_for_a_complete_quiz(local_db):
|
||||
owner = _user("qs")
|
||||
quiz = _quiz(owner)
|
||||
_choice(quiz["uid"])
|
||||
assert store.validation_errors(quiz["uid"]) == []
|
||||
|
||||
|
||||
def test_validation_errors_flags_a_free_text_without_criteria(local_db):
|
||||
owner = _user("qs")
|
||||
quiz = _quiz(owner)
|
||||
store.add_question(
|
||||
quiz["uid"],
|
||||
{"kind": "free_text", "prompt": "Explain", "points": 1, "options": []},
|
||||
)
|
||||
assert any("reference answer" in problem for problem in store.validation_errors(quiz["uid"]))
|
||||
|
||||
|
||||
def test_validation_errors_flags_a_matching_with_one_pair(local_db):
|
||||
owner = _user("qs")
|
||||
quiz = _quiz(owner)
|
||||
store.add_question(
|
||||
quiz["uid"],
|
||||
{
|
||||
"kind": "matching",
|
||||
"prompt": "Match",
|
||||
"points": 1,
|
||||
"options": [{"label": "a", "match_value": "b", "is_correct": False}],
|
||||
},
|
||||
)
|
||||
assert any("two pairs" in problem for problem in store.validation_errors(quiz["uid"]))
|
||||
|
||||
|
||||
def test_publish_refuses_an_incomplete_quiz(local_db):
|
||||
owner = _user("qs")
|
||||
quiz = _quiz(owner)
|
||||
with pytest.raises(QuizError):
|
||||
store.publish_quiz(quiz["uid"])
|
||||
assert store.get_quiz(quiz["uid"])["status"] == "draft"
|
||||
|
||||
|
||||
def test_publish_sets_the_status_once(local_db):
|
||||
owner = _user("qs")
|
||||
quiz = _quiz(owner)
|
||||
_choice(quiz["uid"])
|
||||
published = store.publish_quiz(quiz["uid"])
|
||||
assert published["status"] == "published"
|
||||
assert published["published_at"]
|
||||
assert published["publish_won"] is True
|
||||
|
||||
|
||||
def test_publishing_twice_does_not_win_twice(local_db):
|
||||
owner = _user("qs")
|
||||
quiz = _quiz(owner)
|
||||
_choice(quiz["uid"])
|
||||
store.publish_quiz(quiz["uid"])
|
||||
again = store.publish_quiz(quiz["uid"])
|
||||
assert again.get("publish_won") is not True
|
||||
|
||||
|
||||
def test_guard_editable_refuses_a_published_quiz(local_db):
|
||||
owner = _user("qs")
|
||||
quiz = _quiz(owner)
|
||||
_choice(quiz["uid"])
|
||||
store.publish_quiz(quiz["uid"])
|
||||
with pytest.raises(QuizError):
|
||||
store.guard_editable(quiz["uid"])
|
||||
|
||||
|
||||
def test_every_mutating_entrypoint_is_frozen_after_publish(local_db):
|
||||
owner = _user("qs")
|
||||
quiz = _quiz(owner)
|
||||
question = _choice(quiz["uid"])
|
||||
store.publish_quiz(quiz["uid"])
|
||||
with pytest.raises(QuizError):
|
||||
store.add_question(quiz["uid"], {"kind": "numeric", "prompt": "x", "options": []})
|
||||
with pytest.raises(QuizError):
|
||||
store.edit_question(quiz["uid"], question["uid"], {"kind": "numeric", "prompt": "x", "options": []})
|
||||
with pytest.raises(QuizError):
|
||||
store.delete_question(quiz["uid"], question["uid"], owner["uid"])
|
||||
with pytest.raises(QuizError):
|
||||
store.reorder_questions(quiz["uid"], [question["uid"]])
|
||||
with pytest.raises(QuizError):
|
||||
store.edit_quiz(quiz["uid"], {"title": "new"})
|
||||
|
||||
|
||||
def test_a_published_quiz_keeps_its_question_bytes(local_db):
|
||||
owner = _user("qs")
|
||||
quiz = _quiz(owner)
|
||||
question = _choice(quiz["uid"])
|
||||
store.publish_quiz(quiz["uid"])
|
||||
with pytest.raises(QuizError):
|
||||
store.edit_question(
|
||||
quiz["uid"], question["uid"], {"kind": "numeric", "prompt": "changed", "options": []}
|
||||
)
|
||||
assert store.list_questions(quiz["uid"])[0]["prompt"] == "Pick one"
|
||||
|
||||
|
||||
def test_can_view_quiz_hides_a_draft_from_everyone_else(local_db):
|
||||
owner = _user("qs")
|
||||
other = _user("qs")
|
||||
quiz = _quiz(owner)
|
||||
assert store.can_view_quiz(quiz, owner) is True
|
||||
assert store.can_view_quiz(quiz, other) is False
|
||||
assert store.can_view_quiz(quiz, None) is False
|
||||
|
||||
|
||||
def test_can_view_quiz_allows_everyone_once_published(local_db):
|
||||
owner = _user("qs")
|
||||
other = _user("qs")
|
||||
quiz = _quiz(owner)
|
||||
_choice(quiz["uid"])
|
||||
store.publish_quiz(quiz["uid"])
|
||||
assert store.can_view_quiz(store.get_quiz(quiz["uid"]), other) is True
|
||||
assert store.can_view_quiz(store.get_quiz(quiz["uid"]), None) is True
|
||||
|
||||
|
||||
def test_starting_twice_returns_the_same_attempt(local_db):
|
||||
owner = _user("qs")
|
||||
player = _user("qs")
|
||||
quiz = _quiz(owner)
|
||||
_choice(quiz["uid"])
|
||||
store.publish_quiz(quiz["uid"])
|
||||
quiz = store.get_quiz(quiz["uid"])
|
||||
first = store.start_attempt(player, quiz)
|
||||
second = store.start_attempt(player, quiz)
|
||||
assert first["uid"] == second["uid"]
|
||||
live = get_table("quiz_attempts").count(
|
||||
user_uid=player["uid"], quiz_uid=quiz["uid"], status="in_progress", deleted_at=None
|
||||
)
|
||||
assert live == 1
|
||||
|
||||
|
||||
def test_starting_materializes_a_blank_answer_per_question(local_db):
|
||||
owner = _user("qs")
|
||||
player = _user("qs")
|
||||
quiz = _quiz(owner)
|
||||
_choice(quiz["uid"])
|
||||
_numeric(quiz["uid"])
|
||||
store.publish_quiz(quiz["uid"])
|
||||
attempt = store.start_attempt(player, store.get_quiz(quiz["uid"]))
|
||||
answers = store.answers_for(attempt["uid"])
|
||||
assert len(answers) == 2
|
||||
assert all(not answer["answered_at"] for answer in answers)
|
||||
|
||||
|
||||
def test_starting_snapshots_the_max_points(local_db):
|
||||
owner = _user("qs")
|
||||
player = _user("qs")
|
||||
quiz = _quiz(owner)
|
||||
_choice(quiz["uid"], points=5)
|
||||
store.publish_quiz(quiz["uid"])
|
||||
attempt = store.start_attempt(player, store.get_quiz(quiz["uid"]))
|
||||
assert attempt["max_points"] == 5
|
||||
|
||||
|
||||
def test_answering_credits_the_attempt(local_db):
|
||||
owner = _user("qs")
|
||||
player = _user("qs")
|
||||
quiz = _quiz(owner)
|
||||
question = _numeric(quiz["uid"], value=7.0, points=4)
|
||||
store.publish_quiz(quiz["uid"])
|
||||
quiz = store.get_quiz(quiz["uid"])
|
||||
attempt = store.start_attempt(player, quiz)
|
||||
answer, updated, result = run_async(
|
||||
store.answer(
|
||||
player, quiz, attempt, question["uid"], {"answer_text": "7", "option_uids": []}
|
||||
)
|
||||
)
|
||||
assert answer["is_correct"] == 1
|
||||
assert float(answer["awarded_points"]) == 4.0
|
||||
assert float(updated["score_points"]) == 4.0
|
||||
assert int(updated["answered_count"]) == 1
|
||||
assert result.graded_by == "auto"
|
||||
|
||||
|
||||
def test_a_second_answer_is_refused_and_credits_nothing(local_db):
|
||||
owner = _user("qs")
|
||||
player = _user("qs")
|
||||
quiz = _quiz(owner)
|
||||
question = _numeric(quiz["uid"], value=7.0, points=4)
|
||||
store.publish_quiz(quiz["uid"])
|
||||
quiz = store.get_quiz(quiz["uid"])
|
||||
attempt = store.start_attempt(player, quiz)
|
||||
run_async(
|
||||
store.answer(
|
||||
player, quiz, attempt, question["uid"], {"answer_text": "7", "option_uids": []}
|
||||
)
|
||||
)
|
||||
with pytest.raises(QuizError):
|
||||
run_async(
|
||||
store.answer(
|
||||
player,
|
||||
quiz,
|
||||
store.get_attempt(attempt["uid"]),
|
||||
question["uid"],
|
||||
{"answer_text": "7", "option_uids": []},
|
||||
)
|
||||
)
|
||||
refreshed = store.get_attempt(attempt["uid"])
|
||||
assert float(refreshed["score_points"]) == 4.0
|
||||
assert int(refreshed["answered_count"]) == 1
|
||||
|
||||
|
||||
def test_finishing_recomputes_the_score_from_the_answers(local_db):
|
||||
owner = _user("qs")
|
||||
player = _user("qs")
|
||||
quiz = _quiz(owner, pass_percent=50)
|
||||
first = _numeric(quiz["uid"], value=1.0, points=2)
|
||||
_numeric(quiz["uid"], value=2.0, points=2)
|
||||
store.publish_quiz(quiz["uid"])
|
||||
quiz = store.get_quiz(quiz["uid"])
|
||||
attempt = store.start_attempt(player, quiz)
|
||||
run_async(
|
||||
store.answer(
|
||||
player, quiz, attempt, first["uid"], {"answer_text": "1", "option_uids": []}
|
||||
)
|
||||
)
|
||||
finished, won = store.finish(quiz, store.get_attempt(attempt["uid"]))
|
||||
assert won is True
|
||||
assert finished["status"] == "completed"
|
||||
assert float(finished["score_points"]) == 2.0
|
||||
assert float(finished["score_percent"]) == 50.0
|
||||
assert int(finished["passed"]) == 1
|
||||
|
||||
|
||||
def test_finishing_twice_wins_only_once(local_db):
|
||||
owner = _user("qs")
|
||||
player = _user("qs")
|
||||
quiz = _quiz(owner)
|
||||
_choice(quiz["uid"])
|
||||
store.publish_quiz(quiz["uid"])
|
||||
quiz = store.get_quiz(quiz["uid"])
|
||||
attempt = store.start_attempt(player, quiz)
|
||||
store.finish(quiz, attempt)
|
||||
_, won = store.finish(quiz, store.get_attempt(attempt["uid"]))
|
||||
assert won is False
|
||||
assert int(store.get_quiz(quiz["uid"])["attempt_count"]) == 1
|
||||
|
||||
|
||||
def test_an_expired_attempt_refuses_further_answers(local_db):
|
||||
owner = _user("qs")
|
||||
player = _user("qs")
|
||||
quiz = _quiz(owner, time_limit_seconds=60)
|
||||
question = _numeric(quiz["uid"])
|
||||
store.publish_quiz(quiz["uid"])
|
||||
quiz = store.get_quiz(quiz["uid"])
|
||||
attempt = store.start_attempt(player, quiz)
|
||||
past = (datetime.now(timezone.utc) - timedelta(seconds=10)).isoformat()
|
||||
get_table("quiz_attempts").update({"uid": attempt["uid"], "expires_at": past}, ["uid"])
|
||||
with pytest.raises(QuizError):
|
||||
run_async(
|
||||
store.answer(
|
||||
player,
|
||||
quiz,
|
||||
store.get_attempt(attempt["uid"]),
|
||||
question["uid"],
|
||||
{"answer_text": "1", "option_uids": []},
|
||||
)
|
||||
)
|
||||
assert store.get_attempt(attempt["uid"])["status"] == "expired"
|
||||
|
||||
|
||||
def test_expiry_happens_lazily_on_a_read(local_db):
|
||||
owner = _user("qs")
|
||||
player = _user("qs")
|
||||
quiz = _quiz(owner, time_limit_seconds=60)
|
||||
_numeric(quiz["uid"])
|
||||
store.publish_quiz(quiz["uid"])
|
||||
quiz = store.get_quiz(quiz["uid"])
|
||||
attempt = store.start_attempt(player, quiz)
|
||||
past = (datetime.now(timezone.utc) - timedelta(seconds=10)).isoformat()
|
||||
get_table("quiz_attempts").update({"uid": attempt["uid"], "expires_at": past}, ["uid"])
|
||||
serialized = store.serialize_attempt(quiz, store.get_attempt(attempt["uid"]), player)
|
||||
assert serialized["status"] == "expired"
|
||||
|
||||
|
||||
def test_the_serializer_withholds_the_answer_key_from_a_player(local_db):
|
||||
owner = _user("qs")
|
||||
player = _user("qs")
|
||||
quiz = _quiz(owner, reveal_answers=0)
|
||||
_choice(quiz["uid"])
|
||||
store.publish_quiz(quiz["uid"])
|
||||
quiz = store.get_quiz(quiz["uid"])
|
||||
attempt = store.start_attempt(player, quiz)
|
||||
payload = store.serialize_attempt(quiz, attempt, player)
|
||||
question = payload["questions"][0]
|
||||
assert "correct_boolean" not in question
|
||||
assert "expected_answer" not in question
|
||||
assert all("is_correct" not in option for option in question["options"])
|
||||
|
||||
|
||||
def test_the_serializer_gives_the_owner_the_answer_key(local_db):
|
||||
owner = _user("qs")
|
||||
quiz = _quiz(owner, reveal_answers=0)
|
||||
_choice(quiz["uid"])
|
||||
store.publish_quiz(quiz["uid"])
|
||||
quiz = store.get_quiz(quiz["uid"])
|
||||
attempt = store.start_attempt(owner, quiz)
|
||||
payload = store.serialize_attempt(quiz, attempt, owner)
|
||||
question = payload["questions"][0]
|
||||
assert any("is_correct" in option for option in question["options"])
|
||||
|
||||
|
||||
def test_the_serializer_reveals_an_answered_question_when_configured(local_db):
|
||||
owner = _user("qs")
|
||||
player = _user("qs")
|
||||
quiz = _quiz(owner, reveal_answers=1)
|
||||
question = _numeric(quiz["uid"], value=3.0)
|
||||
store.publish_quiz(quiz["uid"])
|
||||
quiz = store.get_quiz(quiz["uid"])
|
||||
attempt = store.start_attempt(player, quiz)
|
||||
run_async(
|
||||
store.answer(
|
||||
player, quiz, attempt, question["uid"], {"answer_text": "3", "option_uids": []}
|
||||
)
|
||||
)
|
||||
payload = store.serialize_attempt(quiz, store.get_attempt(attempt["uid"]), player)
|
||||
assert "numeric_value" in payload["questions"][0]
|
||||
|
||||
|
||||
def test_matching_choices_are_exposed_without_the_pairing(local_db):
|
||||
owner = _user("qs")
|
||||
player = _user("qs")
|
||||
quiz = _quiz(owner, reveal_answers=0)
|
||||
store.add_question(
|
||||
quiz["uid"],
|
||||
{
|
||||
"kind": "matching",
|
||||
"prompt": "Match",
|
||||
"points": 1,
|
||||
"options": [
|
||||
{"label": "a", "match_value": "A", "is_correct": False},
|
||||
{"label": "b", "match_value": "B", "is_correct": False},
|
||||
],
|
||||
},
|
||||
)
|
||||
store.publish_quiz(quiz["uid"])
|
||||
quiz = store.get_quiz(quiz["uid"])
|
||||
attempt = store.start_attempt(player, quiz)
|
||||
question = store.serialize_attempt(quiz, attempt, player)["questions"][0]
|
||||
assert sorted(question["match_choices"]) == ["A", "B"]
|
||||
assert all("match_value" not in option for option in question["options"])
|
||||
|
||||
|
||||
def test_the_cascade_removes_every_child_under_one_stamp(local_db):
|
||||
owner = _user("qs")
|
||||
player = _user("qs")
|
||||
quiz = _quiz(owner)
|
||||
_choice(quiz["uid"])
|
||||
store.publish_quiz(quiz["uid"])
|
||||
store.start_attempt(player, store.get_quiz(quiz["uid"]))
|
||||
stamp = datetime.now(timezone.utc).isoformat()
|
||||
store.cascade_questions(quiz["uid"], owner["uid"], stamp)
|
||||
for table in ("quiz_questions", "quiz_options", "quiz_attempts", "quiz_answers"):
|
||||
live = get_table(table).count(quiz_uid=quiz["uid"], deleted_at=None)
|
||||
stamped = get_table(table).count(quiz_uid=quiz["uid"], deleted_at=stamp)
|
||||
assert live == 0
|
||||
assert stamped > 0
|
||||
|
||||
|
||||
def test_prune_attempts_keeps_completed_records(local_db):
|
||||
owner = _user("qs")
|
||||
player = _user("qs")
|
||||
quiz = _quiz(owner)
|
||||
_choice(quiz["uid"])
|
||||
store.publish_quiz(quiz["uid"])
|
||||
quiz = store.get_quiz(quiz["uid"])
|
||||
kept = store.start_attempt(player, quiz)
|
||||
store.finish(quiz, kept)
|
||||
future = (datetime.now(timezone.utc) + timedelta(days=1)).isoformat()
|
||||
store.prune_attempts(future)
|
||||
assert store.get_attempt(kept["uid"]) is not None
|
||||
|
||||
|
||||
def test_prune_attempts_removes_an_abandoned_one(local_db):
|
||||
owner = _user("qs")
|
||||
player = _user("qs")
|
||||
quiz = _quiz(owner)
|
||||
_choice(quiz["uid"])
|
||||
store.publish_quiz(quiz["uid"])
|
||||
attempt = store.start_attempt(player, store.get_quiz(quiz["uid"]))
|
||||
future = (datetime.now(timezone.utc) + timedelta(days=1)).isoformat()
|
||||
store.prune_attempts(future)
|
||||
assert store.get_attempt(attempt["uid"]) is None
|
||||
|
||||
|
||||
def test_the_attempt_order_is_persisted_as_json(local_db):
|
||||
owner = _user("qs")
|
||||
player = _user("qs")
|
||||
quiz = _quiz(owner, shuffle_questions=1)
|
||||
_numeric(quiz["uid"], value=1.0)
|
||||
_numeric(quiz["uid"], value=2.0)
|
||||
_numeric(quiz["uid"], value=3.0)
|
||||
store.publish_quiz(quiz["uid"])
|
||||
attempt = store.start_attempt(player, store.get_quiz(quiz["uid"]))
|
||||
order = json.loads(attempt["question_order"])
|
||||
assert sorted(order) == sorted(store.question_uids(quiz["uid"]))
|
||||
assert store.attempt_order(store.get_attempt(attempt["uid"])) == order
|
||||
Reference in New Issue
Block a user