devplacepy/tests/unit/services/quiz/grading.py

# retoor <retoor@molodetz.nl>

import json

from devplacepy.config import QUIZ_FEEDBACK_MAX_CHARS
from devplacepy.services.quiz import grading, scoring

QUESTION = {
    "uid": "q1",
    "prompt": "Why does a partial index help here?",
    "expected_answer": "It keeps the live IS NULL query off a one bucket index.",
    "grading_criteria": "Accept any answer mentioning the planner.",
    "points": 3,
}


def test_parse_verdict_reads_a_plain_object():
    parsed = grading.parse_verdict('{"correct": true, "score": 0.8}')
    assert parsed == {"correct": True, "score": 0.8}


def test_parse_verdict_tolerates_a_code_fence():
    parsed = grading.parse_verdict('```json\n{"score": 0.5}\n```')
    assert parsed == {"score": 0.5}


def test_parse_verdict_tolerates_surrounding_prose():
    parsed = grading.parse_verdict('Here you go: {"score": 1.0} hope that helps')
    assert parsed == {"score": 1.0}


def test_parse_verdict_returns_none_for_junk():
    assert grading.parse_verdict("not json at all") is None
    assert grading.parse_verdict("") is None


def test_parse_verdict_returns_none_for_a_json_list():
    assert grading.parse_verdict("[1, 2, 3]") is None


def test_build_result_clamps_a_high_score():
    result = grading.build_result({"score": 4.0, "correct": True})
    assert result.score == 1.0


def test_build_result_clamps_a_negative_score():
    result = grading.build_result({"score": -2.0, "correct": False})
    assert result.score == 0.0


def test_build_result_derives_correct_from_the_clamped_score():
    result = grading.build_result({"correct": True, "score": 0.0})
    assert result.is_correct is False


def test_build_result_marks_a_high_score_correct_despite_the_model():
    result = grading.build_result({"correct": False, "score": 0.9})
    assert result.is_correct is True


def test_build_result_falls_back_to_the_boolean_without_a_score():
    assert grading.build_result({"correct": True}).score == 1.0
    assert grading.build_result({"correct": False}).score == 0.0


def test_build_result_handles_an_unparsable_score():
    assert grading.build_result({"correct": True, "score": "lots"}).score == 1.0


def test_build_result_clamps_the_confidence():
    assert grading.build_result({"score": 1.0, "confidence": 9.0}).confidence == 1.0
    assert grading.build_result({"score": 1.0, "confidence": -9.0}).confidence == 0.0


def test_build_result_truncates_and_strips_feedback():
    parsed = {"score": 1.0, "feedback": "<b>nice</b> " + "x" * (QUIZ_FEEDBACK_MAX_CHARS + 50)}
    result = grading.build_result(parsed)
    assert "<b>" not in result.feedback
    assert len(result.feedback) <= QUIZ_FEEDBACK_MAX_CHARS


def test_build_result_always_has_feedback():
    assert grading.build_result({"score": 1.0}).feedback


def test_build_result_is_stamped_as_ai():
    assert grading.build_result({"score": 1.0}).graded_by == "ai"


def test_build_prompt_carries_the_grading_context():
    payload = json.loads(grading.build_prompt(QUESTION, "because the planner"))
    assert payload["reference_answer"] == QUESTION["expected_answer"]
    assert payload["grading_criteria"] == QUESTION["grading_criteria"]
    assert payload["learner_answer"] == "because the planner"


def test_system_prompt_treats_the_answer_as_data():
    assert "DATA" in grading.SYSTEM_PROMPT
    assert "JSON" in grading.SYSTEM_PROMPT


def test_grade_free_text_falls_back_without_an_api_key():
    result = grading.grade_free_text("", QUESTION, "the planner picks a bad index")
    assert result.graded_by == "fallback"
    assert 0.0 <= result.score <= 1.0


def test_grade_free_text_falls_back_when_the_gateway_raises(monkeypatch):
    def boom(*args, **kwargs):
        raise RuntimeError("gateway down")

    monkeypatch.setattr(grading, "gateway_complete", boom)
    result = grading.grade_free_text("key", QUESTION, "the planner picks a bad index")
    assert result.graded_by == "fallback"
    assert "unavailable" in result.feedback


def test_grade_free_text_falls_back_on_an_unreadable_reply(monkeypatch):
    monkeypatch.setattr(grading, "gateway_complete", lambda *a, **k: ("not json", None))
    result = grading.grade_free_text("key", QUESTION, "the planner picks a bad index")
    assert result.graded_by == "fallback"


def test_grade_free_text_uses_a_valid_verdict(monkeypatch):
    reply = json.dumps({"correct": True, "score": 0.75, "feedback": "Close.", "confidence": 0.9})
    monkeypatch.setattr(grading, "gateway_complete", lambda *a, **k: (reply, None))
    result = grading.grade_free_text("key", QUESTION, "the planner")
    assert result.graded_by == "ai"
    assert result.score == 0.75
    assert result.is_correct is True
    assert result.feedback == "Close."


def test_grade_free_text_passes_the_answering_key_through(monkeypatch):
    seen = {}

    def capture(api_key, system, text, timeout):
        seen["api_key"] = api_key
        return json.dumps({"score": 1.0}), None

    monkeypatch.setattr(grading, "gateway_complete", capture)
    grading.grade_free_text("member-key", QUESTION, "answer")
    assert seen["api_key"] == "member-key"


def test_the_fallback_score_is_the_deterministic_overlap():
    expected = scoring.token_overlap_score(QUESTION["expected_answer"], "one bucket index")
    result = grading.grade_free_text("", QUESTION, "one bucket index")
    assert result.score == expected