Quiiz system

2026-07-26 16:46:41 +02:00
parent 7f17d69f5c
commit 4780016980
192 changed files with 15031 additions and 566 deletions
@@ -0,0 +1,270 @@
+# retoor <retoor@molodetz.nl>
+
+import time
+
+
+from tests.api.quizzes._helpers import (
+    CHOICE,
+    FREE_TEXT,
+    JSON,
+    NUMERIC,
+    TRUE_FALSE,
+    answer_correctly,
+    attempt_state,
+    build_published,
+    signup,
+    start_attempt,
+)
+from tests.conftest import BASE_URL
+
+CACHE_VERSION_TTL_SECONDS = 1.5
+
+FILL_BLANK = {
+    "kind": "fill_blank",
+    "prompt": "Name the two pragmas",
+    "points": 2,
+    "options": "First\nSecond",
+    "match_values": "journal_mode\nsynchronous",
+}
+
+ORDERING = {
+    "kind": "ordering",
+    "prompt": "Order the steps",
+    "points": 3,
+    "options": "Open\nRead\nClose",
+}
+
+MATCHING = {
+    "kind": "matching",
+    "prompt": "Match the pragma",
+    "points": 2,
+    "options": "journal_mode\nsynchronous",
+    "match_values": "WAL\nNORMAL",
+}
+
+
+def _single_question(session, question, **settings):
+    slug = build_published(session, questions=(question,), title=f"Graded {question['kind']}", **settings)
+    return slug
+
+
+def _drop_api_key(username):
+    from devplacepy.database import get_table, refresh_snapshot
+    from devplacepy.utils import clear_user_cache
+
+    refresh_snapshot()
+    user = get_table("users").find_one(username=username)
+    get_table("users").update({"uid": user["uid"], "api_key": ""}, ["uid"])
+    clear_user_cache(user["uid"])
+    refresh_snapshot()
+    time.sleep(CACHE_VERSION_TTL_SECONDS)
+
+
+def _answer(session, slug, payload):
+    attempt_uid = start_attempt(session, slug)
+    question = attempt_state(session, slug, attempt_uid)["attempt"]["questions"][0]
+    body = {"question_uid": question["uid"], **payload(question)}
+    response = session.post(
+        f"{BASE_URL}/quizzes/{slug}/attempts/{attempt_uid}/answer", data=body, headers=JSON
+    )
+    return response, question
+
+
+def test_a_single_choice_is_graded_deterministically(app_server, seeded_db):
+    owner, _ = signup()
+    slug = _single_question(owner, CHOICE)
+    player, _ = signup()
+    response, _ = _answer(
+        player, slug, lambda question: {"option_uids": question["options"][1]["uid"]}
+    )
+    answer = response.json()["answer"]
+    assert answer["graded_by"] == "auto"
+    assert answer["is_correct"] is True
+    assert answer["awarded_points"] == 2.0
+
+
+def test_a_wrong_single_choice_awards_nothing(app_server, seeded_db):
+    owner, _ = signup()
+    slug = _single_question(owner, CHOICE)
+    player, _ = signup()
+    response, _ = _answer(
+        player, slug, lambda question: {"option_uids": question["options"][0]["uid"]}
+    )
+    assert response.json()["answer"]["awarded_points"] == 0.0
+
+
+def test_a_true_false_is_graded(app_server, seeded_db):
+    owner, _ = signup()
+    slug = _single_question(owner, TRUE_FALSE)
+    player, _ = signup()
+    response, _ = _answer(player, slug, lambda question: {"answer_text": "true"})
+    assert response.json()["answer"]["is_correct"] is True
+
+
+def test_a_numeric_within_tolerance_is_correct(app_server, seeded_db):
+    owner, _ = signup()
+    slug = _single_question(owner, {**NUMERIC, "numeric_tolerance": "1"})
+    player, _ = signup()
+    response, _ = _answer(player, slug, lambda question: {"answer_text": "1023"})
+    assert response.json()["answer"]["is_correct"] is True
+
+
+def test_a_fill_blank_gives_partial_credit(app_server, seeded_db):
+    owner, _ = signup()
+    slug = _single_question(owner, FILL_BLANK)
+    player, _ = signup()
+    response, _ = _answer(
+        player, slug, lambda question: {"blanks": "journal_mode,wrong"}
+    )
+    answer = response.json()["answer"]
+    assert answer["awarded_points"] == 1.0
+    assert answer["is_correct"] is False
+
+
+def test_a_full_fill_blank_is_correct(app_server, seeded_db):
+    owner, _ = signup()
+    slug = _single_question(owner, FILL_BLANK)
+    player, _ = signup()
+    response, _ = _answer(
+        player, slug, lambda question: {"blanks": "journal_mode,synchronous"}
+    )
+    assert response.json()["answer"]["is_correct"] is True
+
+
+def test_an_ordering_gives_prefix_credit(app_server, seeded_db):
+    owner, _ = signup()
+    slug = _single_question(owner, ORDERING)
+    player, _ = signup()
+
+    def payload(question):
+        uids = [option["uid"] for option in question["options"]]
+        return {"option_uids": ",".join([uids[0], uids[2], uids[1]])}
+
+    response, _ = _answer(player, slug, payload)
+    assert response.json()["answer"]["awarded_points"] == 1.0
+
+
+def test_a_matching_gives_pair_credit(app_server, seeded_db):
+    owner, _ = signup()
+    slug = _single_question(owner, MATCHING)
+    player, _ = signup()
+
+    def payload(question):
+        uids = [option["uid"] for option in question["options"]]
+        return {"option_uids": ",".join(uids), "matches": "WAL,wrong"}
+
+    response, _ = _answer(player, slug, payload)
+    assert response.json()["answer"]["awarded_points"] == 1.0
+
+
+def test_a_matching_exposes_its_right_hand_choices(app_server, seeded_db):
+    owner, _ = signup()
+    slug = _single_question(owner, MATCHING)
+    player, _ = signup()
+    attempt_uid = start_attempt(player, slug)
+    question = attempt_state(player, slug, attempt_uid)["attempt"]["questions"][0]
+    assert sorted(question["match_choices"]) == ["NORMAL", "WAL"]
+    assert all(option["match_value"] is None for option in question["options"])
+
+
+def test_a_free_text_answer_is_still_graded_without_the_gateway(app_server, seeded_db):
+    owner, _ = signup()
+    slug = _single_question(owner, FREE_TEXT)
+    player, player_name = signup()
+    _drop_api_key(player_name)
+    response, _ = _answer(
+        player,
+        slug,
+        lambda question: {"answer_text": "the planner picks a bad index for live rows"},
+    )
+    assert response.status_code == 200
+    assert response.json()["answer"]["answered"] is True
+
+
+def test_the_degraded_grade_is_stamped_as_a_fallback(app_server, seeded_db):
+    owner, _ = signup()
+    slug = _single_question(owner, FREE_TEXT)
+    player, player_name = signup()
+    _drop_api_key(player_name)
+    response, _ = _answer(
+        player, slug, lambda question: {"answer_text": "a one bucket index for live rows"}
+    )
+    assert response.json()["answer"]["graded_by"] == "fallback"
+
+
+def test_the_fallback_explains_itself_to_the_learner(app_server, seeded_db):
+    owner, _ = signup()
+    slug = _single_question(owner, FREE_TEXT)
+    player, player_name = signup()
+    _drop_api_key(player_name)
+    response, _ = _answer(player, slug, lambda question: {"answer_text": "a one bucket index"})
+    assert "unavailable" in response.json()["answer"]["feedback"]
+
+
+def test_the_fallback_score_stays_within_the_question_points(app_server, seeded_db):
+    owner, _ = signup()
+    slug = _single_question(owner, FREE_TEXT)
+    player, player_name = signup()
+    _drop_api_key(player_name)
+    response, _ = _answer(
+        player,
+        slug,
+        lambda question: {"answer_text": "It keeps the live query off a one bucket index."},
+    )
+    answer = response.json()["answer"]
+    assert 0.0 <= answer["awarded_points"] <= 3.0
+    assert 0.0 <= answer["confidence"] <= 1.0
+
+
+def test_an_empty_free_text_answer_scores_zero(app_server, seeded_db):
+    owner, _ = signup()
+    slug = _single_question(owner, FREE_TEXT)
+    player, player_name = signup()
+    _drop_api_key(player_name)
+    response, _ = _answer(player, slug, lambda question: {"answer_text": "completely unrelated"})
+    assert response.json()["answer"]["awarded_points"] == 0.0
+
+
+def test_the_results_page_surfaces_the_fallback_note(app_server, seeded_db):
+    owner, _ = signup()
+    slug = _single_question(owner, FREE_TEXT, allow_review="1")
+    player, player_name = signup()
+    _drop_api_key(player_name)
+    attempt_uid = start_attempt(player, slug)
+    question = attempt_state(player, slug, attempt_uid)["attempt"]["questions"][0]
+    player.post(
+        f"{BASE_URL}/quizzes/{slug}/attempts/{attempt_uid}/answer",
+        data={"question_uid": question["uid"], "answer_text": "a one bucket index"},
+        headers=JSON,
+    )
+    player.post(f"{BASE_URL}/quizzes/{slug}/attempts/{attempt_uid}/finish", headers=JSON)
+    payload = player.get(
+        f"{BASE_URL}/quizzes/{slug}/attempts/{attempt_uid}/results", headers=JSON
+    ).json()
+    assert payload["fallback_count"] == 1
+    page = player.get(f"{BASE_URL}/quizzes/{slug}/attempts/{attempt_uid}/results").text
+    assert "quiz-grade-fallback" in page
+
+
+def test_a_degraded_grade_records_a_system_audit_failure(app_server, seeded_db):
+    from devplacepy.database import get_table, refresh_snapshot
+
+    owner, _ = signup()
+    slug = _single_question(owner, FREE_TEXT)
+    player, player_name = signup()
+    _drop_api_key(player_name)
+    _answer(player, slug, lambda question: {"answer_text": "a one bucket index"})
+    refresh_snapshot()
+    count = get_table("audit_log").count(event_key="quiz.grade.failed", result="failure")
+    assert count >= 1
+
+
+def test_only_free_text_ever_reaches_the_grader(app_server, seeded_db):
+    owner, _ = signup()
+    slug = build_published(owner, questions=(CHOICE, NUMERIC, TRUE_FALSE), title="No gateway quiz")
+    player, _ = signup()
+    attempt_uid = start_attempt(player, slug)
+    state = attempt_state(player, slug, attempt_uid)
+    for question in state["attempt"]["questions"]:
+        payload = answer_correctly(player, slug, attempt_uid, question).json()
+        assert payload["answer"]["graded_by"] == "auto"