# retoor import time from tests.api.quizzes._helpers import ( CHOICE, FREE_TEXT, JSON, NUMERIC, TRUE_FALSE, answer_correctly, attempt_state, build_published, signup, start_attempt, ) from tests.conftest import BASE_URL CACHE_VERSION_TTL_SECONDS = 1.5 FILL_BLANK = { "kind": "fill_blank", "prompt": "Name the two pragmas", "points": 2, "options": "First\nSecond", "match_values": "journal_mode\nsynchronous", } ORDERING = { "kind": "ordering", "prompt": "Order the steps", "points": 3, "options": "Open\nRead\nClose", } MATCHING = { "kind": "matching", "prompt": "Match the pragma", "points": 2, "options": "journal_mode\nsynchronous", "match_values": "WAL\nNORMAL", } def _single_question(session, question, **settings): slug = build_published(session, questions=(question,), title=f"Graded {question['kind']}", **settings) return slug def _drop_api_key(username): from devplacepy.database import get_table, refresh_snapshot from devplacepy.utils import clear_user_cache refresh_snapshot() user = get_table("users").find_one(username=username) get_table("users").update({"uid": user["uid"], "api_key": ""}, ["uid"]) clear_user_cache(user["uid"]) refresh_snapshot() time.sleep(CACHE_VERSION_TTL_SECONDS) def _answer(session, slug, payload): attempt_uid = start_attempt(session, slug) question = attempt_state(session, slug, attempt_uid)["attempt"]["questions"][0] body = {"question_uid": question["uid"], **payload(question)} response = session.post( f"{BASE_URL}/quizzes/{slug}/attempts/{attempt_uid}/answer", data=body, headers=JSON ) return response, question def test_a_single_choice_is_graded_deterministically(app_server, seeded_db): owner, _ = signup() slug = _single_question(owner, CHOICE) player, _ = signup() response, _ = _answer( player, slug, lambda question: {"option_uids": question["options"][1]["uid"]} ) answer = response.json()["answer"] assert answer["graded_by"] == "auto" assert answer["is_correct"] is True assert answer["awarded_points"] == 2.0 def test_a_wrong_single_choice_awards_nothing(app_server, seeded_db): owner, _ = signup() slug = _single_question(owner, CHOICE) player, _ = signup() response, _ = _answer( player, slug, lambda question: {"option_uids": question["options"][0]["uid"]} ) assert response.json()["answer"]["awarded_points"] == 0.0 def test_a_true_false_is_graded(app_server, seeded_db): owner, _ = signup() slug = _single_question(owner, TRUE_FALSE) player, _ = signup() response, _ = _answer(player, slug, lambda question: {"answer_text": "true"}) assert response.json()["answer"]["is_correct"] is True def test_a_numeric_within_tolerance_is_correct(app_server, seeded_db): owner, _ = signup() slug = _single_question(owner, {**NUMERIC, "numeric_tolerance": "1"}) player, _ = signup() response, _ = _answer(player, slug, lambda question: {"answer_text": "1023"}) assert response.json()["answer"]["is_correct"] is True def test_a_fill_blank_gives_partial_credit(app_server, seeded_db): owner, _ = signup() slug = _single_question(owner, FILL_BLANK) player, _ = signup() response, _ = _answer( player, slug, lambda question: {"blanks": "journal_mode,wrong"} ) answer = response.json()["answer"] assert answer["awarded_points"] == 1.0 assert answer["is_correct"] is False def test_a_full_fill_blank_is_correct(app_server, seeded_db): owner, _ = signup() slug = _single_question(owner, FILL_BLANK) player, _ = signup() response, _ = _answer( player, slug, lambda question: {"blanks": "journal_mode,synchronous"} ) assert response.json()["answer"]["is_correct"] is True def test_an_ordering_gives_prefix_credit(app_server, seeded_db): owner, _ = signup() slug = _single_question(owner, ORDERING) player, _ = signup() def payload(question): uids = [option["uid"] for option in question["options"]] return {"option_uids": ",".join([uids[0], uids[2], uids[1]])} response, _ = _answer(player, slug, payload) assert response.json()["answer"]["awarded_points"] == 1.0 def test_a_matching_gives_pair_credit(app_server, seeded_db): owner, _ = signup() slug = _single_question(owner, MATCHING) player, _ = signup() def payload(question): uids = [option["uid"] for option in question["options"]] return {"option_uids": ",".join(uids), "matches": "WAL,wrong"} response, _ = _answer(player, slug, payload) assert response.json()["answer"]["awarded_points"] == 1.0 def test_a_matching_exposes_its_right_hand_choices(app_server, seeded_db): owner, _ = signup() slug = _single_question(owner, MATCHING) player, _ = signup() attempt_uid = start_attempt(player, slug) question = attempt_state(player, slug, attempt_uid)["attempt"]["questions"][0] assert sorted(question["match_choices"]) == ["NORMAL", "WAL"] assert all(option["match_value"] is None for option in question["options"]) def test_a_free_text_answer_is_still_graded_without_the_gateway(app_server, seeded_db): owner, _ = signup() slug = _single_question(owner, FREE_TEXT) player, player_name = signup() _drop_api_key(player_name) response, _ = _answer( player, slug, lambda question: {"answer_text": "the planner picks a bad index for live rows"}, ) assert response.status_code == 200 assert response.json()["answer"]["answered"] is True def test_the_degraded_grade_is_stamped_as_a_fallback(app_server, seeded_db): owner, _ = signup() slug = _single_question(owner, FREE_TEXT) player, player_name = signup() _drop_api_key(player_name) response, _ = _answer( player, slug, lambda question: {"answer_text": "a one bucket index for live rows"} ) assert response.json()["answer"]["graded_by"] == "fallback" def test_the_fallback_explains_itself_to_the_learner(app_server, seeded_db): owner, _ = signup() slug = _single_question(owner, FREE_TEXT) player, player_name = signup() _drop_api_key(player_name) response, _ = _answer(player, slug, lambda question: {"answer_text": "a one bucket index"}) assert "unavailable" in response.json()["answer"]["feedback"] def test_the_fallback_score_stays_within_the_question_points(app_server, seeded_db): owner, _ = signup() slug = _single_question(owner, FREE_TEXT) player, player_name = signup() _drop_api_key(player_name) response, _ = _answer( player, slug, lambda question: {"answer_text": "It keeps the live query off a one bucket index."}, ) answer = response.json()["answer"] assert 0.0 <= answer["awarded_points"] <= 3.0 assert 0.0 <= answer["confidence"] <= 1.0 def test_an_empty_free_text_answer_scores_zero(app_server, seeded_db): owner, _ = signup() slug = _single_question(owner, FREE_TEXT) player, player_name = signup() _drop_api_key(player_name) response, _ = _answer(player, slug, lambda question: {"answer_text": "completely unrelated"}) assert response.json()["answer"]["awarded_points"] == 0.0 def test_the_results_page_surfaces_the_fallback_note(app_server, seeded_db): owner, _ = signup() slug = _single_question(owner, FREE_TEXT, allow_review="1") player, player_name = signup() _drop_api_key(player_name) attempt_uid = start_attempt(player, slug) question = attempt_state(player, slug, attempt_uid)["attempt"]["questions"][0] player.post( f"{BASE_URL}/quizzes/{slug}/attempts/{attempt_uid}/answer", data={"question_uid": question["uid"], "answer_text": "a one bucket index"}, headers=JSON, ) player.post(f"{BASE_URL}/quizzes/{slug}/attempts/{attempt_uid}/finish", headers=JSON) payload = player.get( f"{BASE_URL}/quizzes/{slug}/attempts/{attempt_uid}/results", headers=JSON ).json() assert payload["fallback_count"] == 1 page = player.get(f"{BASE_URL}/quizzes/{slug}/attempts/{attempt_uid}/results").text assert "quiz-grade-fallback" in page def test_a_degraded_grade_records_a_system_audit_failure(app_server, seeded_db): from devplacepy.database import get_table, refresh_snapshot owner, _ = signup() slug = _single_question(owner, FREE_TEXT) player, player_name = signup() _drop_api_key(player_name) _answer(player, slug, lambda question: {"answer_text": "a one bucket index"}) refresh_snapshot() count = get_table("audit_log").count(event_key="quiz.grade.failed", result="failure") assert count >= 1 def test_only_free_text_ever_reaches_the_grader(app_server, seeded_db): owner, _ = signup() slug = build_published(owner, questions=(CHOICE, NUMERIC, TRUE_FALSE), title="No gateway quiz") player, _ = signup() attempt_uid = start_attempt(player, slug) state = attempt_state(player, slug, attempt_uid) for question in state["attempt"]["questions"]: payload = answer_correctly(player, slug, attempt_uid, question).json() assert payload["answer"]["graded_by"] == "auto"