# retoor <retoor@molodetz.nl>
import time
from tests.api.quizzes._helpers import (
CHOICE,
FREE_TEXT,
JSON,
NUMERIC,
TRUE_FALSE,
answer_correctly,
attempt_state,
build_published,
signup,
start_attempt,
)
from tests.conftest import BASE_URL
CACHE_VERSION_TTL_SECONDS = 1.5
FILL_BLANK = {
"kind": "fill_blank",
"prompt": "Name the two pragmas",
"points": 2,
"options": "First\nSecond",
"match_values": "journal_mode\nsynchronous",
}
ORDERING = {
"kind": "ordering",
"prompt": "Order the steps",
"points": 3,
"options": "Open\nRead\nClose",
}
MATCHING = {
"kind": "matching",
"prompt": "Match the pragma",
"points": 2,
"options": "journal_mode\nsynchronous",
"match_values": "WAL\nNORMAL",
}
def _single_question(session, question, **settings):
slug = build_published(session, questions=(question,), title=f"Graded {question['kind']}", **settings)
return slug
def _drop_api_key(username):
from devplacepy.database import get_table, refresh_snapshot
from devplacepy.utils import clear_user_cache
refresh_snapshot()
user = get_table("users").find_one(username=username)
get_table("users").update({"uid": user["uid"], "api_key": ""}, ["uid"])
clear_user_cache(user["uid"])
refresh_snapshot()
time.sleep(CACHE_VERSION_TTL_SECONDS)
def _answer(session, slug, payload):
attempt_uid = start_attempt(session, slug)
question = attempt_state(session, slug, attempt_uid)["attempt"]["questions"][0]
body = {"question_uid": question["uid"], **payload(question)}
response = session.post(
f"{BASE_URL}/quizzes/{slug}/attempts/{attempt_uid}/answer", data=body, headers=JSON
)
return response, question
def test_a_single_choice_is_graded_deterministically(app_server, seeded_db):
owner, _ = signup()
slug = _single_question(owner, CHOICE)
player, _ = signup()
response, _ = _answer(
player, slug, lambda question: {"option_uids": question["options"][1]["uid"]}
)
answer = response.json()["answer"]
assert answer["graded_by"] == "auto"
assert answer["is_correct"] is True
assert answer["awarded_points"] == 2.0
def test_a_wrong_single_choice_awards_nothing(app_server, seeded_db):
owner, _ = signup()
slug = _single_question(owner, CHOICE)
player, _ = signup()
response, _ = _answer(
player, slug, lambda question: {"option_uids": question["options"][0]["uid"]}
)
assert response.json()["answer"]["awarded_points"] == 0.0
def test_a_true_false_is_graded(app_server, seeded_db):
owner, _ = signup()
slug = _single_question(owner, TRUE_FALSE)
player, _ = signup()
response, _ = _answer(player, slug, lambda question: {"answer_text": "true"})
assert response.json()["answer"]["is_correct"] is True
def test_a_numeric_within_tolerance_is_correct(app_server, seeded_db):
owner, _ = signup()
slug = _single_question(owner, {**NUMERIC, "numeric_tolerance": "1"})
player, _ = signup()
response, _ = _answer(player, slug, lambda question: {"answer_text": "1023"})
assert response.json()["answer"]["is_correct"] is True
def test_a_fill_blank_gives_partial_credit(app_server, seeded_db):
owner, _ = signup()
slug = _single_question(owner, FILL_BLANK)
player, _ = signup()
response, _ = _answer(
player, slug, lambda question: {"blanks": "journal_mode,wrong"}
)
answer = response.json()["answer"]
assert answer["awarded_points"] == 1.0
assert answer["is_correct"] is False
def test_a_full_fill_blank_is_correct(app_server, seeded_db):
owner, _ = signup()
slug = _single_question(owner, FILL_BLANK)
player, _ = signup()
response, _ = _answer(
player, slug, lambda question: {"blanks": "journal_mode,synchronous"}
)
assert response.json()["answer"]["is_correct"] is True
def test_an_ordering_gives_prefix_credit(app_server, seeded_db):
owner, _ = signup()
slug = _single_question(owner, ORDERING)
player, _ = signup()
def payload(question):
uids = [option["uid"] for option in question["options"]]
return {"option_uids": ",".join([uids[0], uids[2], uids[1]])}
response, _ = _answer(player, slug, payload)
assert response.json()["answer"]["awarded_points"] == 1.0
def test_a_matching_gives_pair_credit(app_server, seeded_db):
owner, _ = signup()
slug = _single_question(owner, MATCHING)
player, _ = signup()
def payload(question):
uids = [option["uid"] for option in question["options"]]
return {"option_uids": ",".join(uids), "matches": "WAL,wrong"}
response, _ = _answer(player, slug, payload)
assert response.json()["answer"]["awarded_points"] == 1.0
def test_a_matching_exposes_its_right_hand_choices(app_server, seeded_db):
owner, _ = signup()
slug = _single_question(owner, MATCHING)
player, _ = signup()
attempt_uid = start_attempt(player, slug)
question = attempt_state(player, slug, attempt_uid)["attempt"]["questions"][0]
assert sorted(question["match_choices"]) == ["NORMAL", "WAL"]
assert all(option["match_value"] is None for option in question["options"])
def test_a_free_text_answer_is_still_graded_without_the_gateway(app_server, seeded_db):
owner, _ = signup()
slug = _single_question(owner, FREE_TEXT)
player, player_name = signup()
_drop_api_key(player_name)
response, _ = _answer(
player,
slug,
lambda question: {"answer_text": "the planner picks a bad index for live rows"},
)
assert response.status_code == 200
assert response.json()["answer"]["answered"] is True
def test_the_degraded_grade_is_stamped_as_a_fallback(app_server, seeded_db):
owner, _ = signup()
slug = _single_question(owner, FREE_TEXT)
player, player_name = signup()
_drop_api_key(player_name)
response, _ = _answer(
player, slug, lambda question: {"answer_text": "a one bucket index for live rows"}
)
assert response.json()["answer"]["graded_by"] == "fallback"
def test_the_fallback_explains_itself_to_the_learner(app_server, seeded_db):
owner, _ = signup()
slug = _single_question(owner, FREE_TEXT)
player, player_name = signup()
_drop_api_key(player_name)
response, _ = _answer(player, slug, lambda question: {"answer_text": "a one bucket index"})
assert "unavailable" in response.json()["answer"]["feedback"]
def test_the_fallback_score_stays_within_the_question_points(app_server, seeded_db):
owner, _ = signup()
slug = _single_question(owner, FREE_TEXT)
player, player_name = signup()
_drop_api_key(player_name)
response, _ = _answer(
player,
slug,
lambda question: {"answer_text": "It keeps the live query off a one bucket index."},
)
answer = response.json()["answer"]
assert 0.0 <= answer["awarded_points"] <= 3.0
assert 0.0 <= answer["confidence"] <= 1.0
def test_an_empty_free_text_answer_scores_zero(app_server, seeded_db):
owner, _ = signup()
slug = _single_question(owner, FREE_TEXT)
player, player_name = signup()
_drop_api_key(player_name)
response, _ = _answer(player, slug, lambda question: {"answer_text": "completely unrelated"})
assert response.json()["answer"]["awarded_points"] == 0.0
def test_the_results_page_surfaces_the_fallback_note(app_server, seeded_db):
owner, _ = signup()
slug = _single_question(owner, FREE_TEXT, allow_review="1")
player, player_name = signup()
_drop_api_key(player_name)
attempt_uid = start_attempt(player, slug)
question = attempt_state(player, slug, attempt_uid)["attempt"]["questions"][0]
player.post(
f"{BASE_URL}/quizzes/{slug}/attempts/{attempt_uid}/answer",
data={"question_uid": question["uid"], "answer_text": "a one bucket index"},
headers=JSON,
)
player.post(f"{BASE_URL}/quizzes/{slug}/attempts/{attempt_uid}/finish", headers=JSON)
payload = player.get(
f"{BASE_URL}/quizzes/{slug}/attempts/{attempt_uid}/results", headers=JSON
).json()
assert payload["fallback_count"] == 1
page = player.get(f"{BASE_URL}/quizzes/{slug}/attempts/{attempt_uid}/results").text
assert "quiz-grade-fallback" in page
def test_a_degraded_grade_records_a_system_audit_failure(app_server, seeded_db):
from devplacepy.database import get_table, refresh_snapshot
owner, _ = signup()
slug = _single_question(owner, FREE_TEXT)
player, player_name = signup()
_drop_api_key(player_name)
_answer(player, slug, lambda question: {"answer_text": "a one bucket index"})
refresh_snapshot()
count = get_table("audit_log").count(event_key="quiz.grade.failed", result="failure")
assert count >= 1
def test_only_free_text_ever_reaches_the_grader(app_server, seeded_db):
owner, _ = signup()
slug = build_published(owner, questions=(CHOICE, NUMERIC, TRUE_FALSE), title="No gateway quiz")
player, _ = signup()
attempt_uid = start_attempt(player, slug)
state = attempt_state(player, slug, attempt_uid)
for question in state["attempt"]["questions"]:
payload = answer_correctly(player, slug, attempt_uid, question).json()
assert payload["answer"]["graded_by"] == "auto"
Reference in New Issue View Git Blame Copy Permalink