devplacepy/tests/unit/services/quiz/scoring.py

# retoor <retoor@molodetz.nl>

import json
from datetime import datetime, timedelta, timezone

import pytest

from devplacepy.services.quiz import scoring

OPTIONS = [
    {"uid": "o0", "position": 0, "label": "L0", "match_value": "M0", "is_correct": 1},
    {"uid": "o1", "position": 1, "label": "L1", "match_value": "M1", "is_correct": 1},
    {"uid": "o2", "position": 2, "label": "L2", "match_value": "M2", "is_correct": 0},
    {"uid": "o3", "position": 3, "label": "L3", "match_value": "M3", "is_correct": 0},
]

NOW = datetime(2026, 7, 25, 12, 0, 0, tzinfo=timezone.utc)


def question(kind, **extra):
    base = {
        "uid": "q1",
        "kind": kind,
        "points": 3,
        "correct_boolean": 0,
        "numeric_value": 0.0,
        "numeric_tolerance": 0.0,
        "case_sensitive": 0,
    }
    base.update(extra)
    return base


def test_question_kinds_are_the_eight_documented_keys():
    assert scoring.KIND_KEYS == (
        "single_choice",
        "multiple_choice",
        "true_false",
        "free_text",
        "fill_blank",
        "numeric",
        "ordering",
        "matching",
    )


def test_only_free_text_is_ai_graded():
    ai = [kind.key for kind in scoring.QUESTION_KINDS if kind.graded_by == "ai"]
    assert ai == ["free_text"]


def test_awarded_points_is_monotonic_and_bounded():
    for points in (1, 7, 100):
        previous = -1.0
        for step in range(0, 201):
            awarded = scoring.awarded_points(points, step / 200.0)
            assert 0.0 <= awarded <= points
            assert awarded >= previous
            previous = awarded


def test_awarded_points_clamps_outside_the_unit_interval():
    assert scoring.awarded_points(5, 4.0) == 5
    assert scoring.awarded_points(5, -4.0) == 0.0


def test_score_percent_stays_within_bounds():
    for max_points in (0, 1, 14, 199):
        for score_points in (0, 1, 14, 400):
            assert 0.0 <= scoring.score_percent(score_points, max_points) <= 100.0


def test_score_percent_is_zero_without_max_points():
    assert scoring.score_percent(10, 0) == 0.0


def test_score_percent_is_exactly_full_at_max():
    assert scoring.score_percent(14, 14) == 100.0


def test_passed_needs_a_pass_mark():
    assert scoring.passed(100.0, 0) is False
    assert scoring.passed(70.0, 70) is True
    assert scoring.passed(69.9, 70) is False


def test_single_choice_grades_the_correct_option():
    result = scoring.grade_answer(question("single_choice"), OPTIONS, {"option_uids": ["o0"]})
    assert result.score == 1.0 and result.is_correct and result.graded_by == "auto"


def test_single_choice_rejects_more_than_one_pick():
    result = scoring.grade_answer(question("single_choice"), OPTIONS, {"option_uids": ["o0", "o1"]})
    assert result.score == 0.0


def test_single_choice_rejects_an_unknown_option():
    result = scoring.grade_answer(question("single_choice"), OPTIONS, {"option_uids": ["nope"]})
    assert result.score == 0.0


def test_multiple_choice_gives_partial_credit():
    result = scoring.grade_answer(question("multiple_choice"), OPTIONS, {"option_uids": ["o0"]})
    assert result.score == 0.5 and not result.is_correct


def test_multiple_choice_penalises_wrong_picks_but_never_below_zero():
    result = scoring.grade_answer(
        question("multiple_choice"), OPTIONS, {"option_uids": ["o2", "o3"]}
    )
    assert result.score == 0.0


def test_multiple_choice_is_correct_only_on_the_exact_set():
    result = scoring.grade_answer(
        question("multiple_choice"), OPTIONS, {"option_uids": ["o0", "o1"]}
    )
    assert result.score == 1.0 and result.is_correct


def test_multiple_choice_ignores_duplicate_picks():
    result = scoring.grade_answer(
        question("multiple_choice"), OPTIONS, {"option_uids": ["o0", "o0", "o1"]}
    )
    assert result.score == 1.0


def test_true_false_matches_the_stored_boolean():
    prompt = question("true_false", correct_boolean=1)
    assert scoring.grade_answer(prompt, [], {"answer_text": "true"}).score == 1.0
    assert scoring.grade_answer(prompt, [], {"answer_text": "false"}).score == 0.0


def test_true_false_without_an_answer_scores_zero():
    assert scoring.grade_answer(question("true_false"), [], {"answer_text": ""}).score == 0.0


def test_free_text_raises_for_the_ai_grader():
    with pytest.raises(scoring.NeedsAiGrading):
        scoring.grade_answer(question("free_text"), [], {"answer_text": "anything"})


def test_numeric_accepts_within_tolerance():
    prompt = question("numeric", numeric_value=10.0, numeric_tolerance=0.5)
    assert scoring.grade_answer(prompt, [], {"answer_text": "10.4"}).score == 1.0
    assert scoring.grade_answer(prompt, [], {"answer_text": "10.6"}).score == 0.0


def test_numeric_accepts_a_comma_decimal_separator():
    prompt = question("numeric", numeric_value=1.5, numeric_tolerance=0.0)
    assert scoring.grade_answer(prompt, [], {"answer_text": "1,5"}).score == 1.0


def test_numeric_rejects_a_non_number():
    prompt = question("numeric", numeric_value=1.0)
    assert scoring.grade_answer(prompt, [], {"answer_text": "ten"}).score == 0.0


def test_fill_blank_scores_per_blank():
    prompt = question("fill_blank")
    payload = json.dumps(["M0", "wrong", "M2", "M3"])
    result = scoring.grade_answer(prompt, OPTIONS, {"answer_text": payload})
    assert result.score == 0.75


def test_fill_blank_normalizes_whitespace_and_case_by_default():
    prompt = question("fill_blank")
    payload = json.dumps(["  m0 ", "M1", "M2", "M3"])
    assert scoring.grade_answer(prompt, OPTIONS, {"answer_text": payload}).score == 1.0


def test_fill_blank_honours_case_sensitivity():
    prompt = question("fill_blank", case_sensitive=1)
    payload = json.dumps(["m0", "M1", "M2", "M3"])
    assert scoring.grade_answer(prompt, OPTIONS, {"answer_text": payload}).score == 0.75


def test_ordering_scores_the_longest_correct_prefix():
    prompt = question("ordering")
    order = ["o0", "o1", "o3", "o2"]
    assert scoring.grade_answer(prompt, OPTIONS, {"option_uids": order}).score == 0.5


def test_ordering_is_correct_on_the_exact_sequence():
    prompt = question("ordering")
    order = ["o0", "o1", "o2", "o3"]
    result = scoring.grade_answer(prompt, OPTIONS, {"option_uids": order})
    assert result.score == 1.0 and result.is_correct


def test_matching_scores_per_pair():
    prompt = question("matching")
    payload = json.dumps({"o0": "M0", "o1": "M1", "o2": "wrong", "o3": "wrong"})
    assert scoring.grade_answer(prompt, OPTIONS, {"answer_text": payload}).score == 0.5


def test_every_kind_survives_a_malformed_submission():
    malformed = [{}, {"answer_text": None, "option_uids": None}, {"answer_text": "junk"}]
    for kind in scoring.KIND_KEYS:
        if kind == "free_text":
            continue
        for submission in malformed:
            result = scoring.grade_answer(question(kind), OPTIONS, submission)
            assert 0.0 <= result.score <= 1.0


def test_an_unknown_kind_scores_zero_instead_of_raising():
    assert scoring.grade_answer(question("telepathy"), OPTIONS, {}).score == 0.0


def test_question_order_is_a_stable_permutation():
    uids = [f"u{index}" for index in range(9)]
    order = scoring.question_order(uids, True, "attempt-1")
    assert sorted(order) == sorted(uids)
    assert order == scoring.question_order(uids, True, "attempt-1")


def test_question_order_differs_per_seed():
    uids = [f"u{index}" for index in range(12)]
    orders = {tuple(scoring.question_order(uids, True, f"a{index}")) for index in range(40)}
    assert len(orders) > 1


def test_question_order_is_untouched_without_shuffle():
    uids = ["a", "b", "c"]
    assert scoring.question_order(uids, False, "seed") == uids


def test_option_order_is_a_stable_permutation():
    uids = ["o0", "o1", "o2", "o3"]
    order = scoring.option_order(uids, True, "answer-1")
    assert sorted(order) == sorted(uids)
    assert order == scoring.option_order(uids, True, "answer-1")


def test_token_overlap_scores_a_perfect_match():
    assert scoring.token_overlap_score("write ahead logging", "write ahead logging") == 1.0


def test_token_overlap_scores_an_empty_answer_zero():
    assert scoring.token_overlap_score("write ahead logging", "") == 0.0


def test_token_overlap_stays_within_bounds():
    score = scoring.token_overlap_score("alpha beta gamma", "alpha delta")
    assert 0.0 < score < 1.0


def test_attempt_expires_at_is_empty_without_a_limit():
    assert scoring.attempt_expires_at(NOW.isoformat(), 0) == ""


def test_attempt_expires_at_adds_the_limit():
    expires = scoring.attempt_expires_at(NOW.isoformat(), 900)
    assert expires == (NOW + timedelta(seconds=900)).isoformat()


def test_is_expired_reads_the_stored_deadline():
    expires = scoring.attempt_expires_at(NOW.isoformat(), 60)
    assert scoring.is_expired(expires, NOW) is False
    assert scoring.is_expired(expires, NOW + timedelta(seconds=61)) is True


def test_is_expired_is_false_without_a_deadline():
    assert scoring.is_expired("", NOW) is False


def test_remaining_seconds_never_goes_negative():
    expires = scoring.attempt_expires_at(NOW.isoformat(), 60)
    assert scoring.remaining_seconds(expires, NOW) == 60
    assert scoring.remaining_seconds(expires, NOW + timedelta(seconds=600)) == 0


def test_fallback_result_is_stamped_and_explains_itself():
    result = scoring.fallback_result("write ahead logging", "write ahead logging", "gateway down")
    assert result.graded_by == "fallback"
    assert result.confidence == 0.0
    assert "unavailable" in result.feedback