forked from retoor/devplacepy
Add openai_gateway/apps.py as the single list of internal gateway apps and send X-OpenRouter-Title (plus legacy X-Title and HTTP-Referer) per app to every upstream. Label the Devii agent, bots, modifier, quiz grading, SEO metadata, community and sync callers. Shorten two app references that exceeded the 30 character limit and were recorded as default. Also includes the pending Muse Code, route health and Responses work. Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
163 lines
5.8 KiB
Python
163 lines
5.8 KiB
Python
# retoor <retoor@molodetz.nl>
|
|
|
|
import json
|
|
|
|
from devplacepy.config import QUIZ_FEEDBACK_MAX_CHARS
|
|
from devplacepy.services.quiz import grading, scoring
|
|
|
|
QUESTION = {
|
|
"uid": "q1",
|
|
"prompt": "Why does a partial index help here?",
|
|
"expected_answer": "It keeps the live IS NULL query off a one bucket index.",
|
|
"grading_criteria": "Accept any answer mentioning the planner.",
|
|
"points": 3,
|
|
}
|
|
|
|
|
|
def test_parse_verdict_reads_a_plain_object():
|
|
parsed = grading.parse_verdict('{"correct": true, "score": 0.8}')
|
|
assert parsed == {"correct": True, "score": 0.8}
|
|
|
|
|
|
def test_parse_verdict_tolerates_a_code_fence():
|
|
parsed = grading.parse_verdict('```json\n{"score": 0.5}\n```')
|
|
assert parsed == {"score": 0.5}
|
|
|
|
|
|
def test_parse_verdict_tolerates_surrounding_prose():
|
|
parsed = grading.parse_verdict('Here you go: {"score": 1.0} hope that helps')
|
|
assert parsed == {"score": 1.0}
|
|
|
|
|
|
def test_parse_verdict_returns_none_for_junk():
|
|
assert grading.parse_verdict("not json at all") is None
|
|
assert grading.parse_verdict("") is None
|
|
|
|
|
|
def test_parse_verdict_returns_none_for_a_json_list():
|
|
assert grading.parse_verdict("[1, 2, 3]") is None
|
|
|
|
|
|
def test_build_result_clamps_a_high_score():
|
|
result = grading.build_result({"score": 4.0, "correct": True})
|
|
assert result.score == 1.0
|
|
|
|
|
|
def test_build_result_clamps_a_negative_score():
|
|
result = grading.build_result({"score": -2.0, "correct": False})
|
|
assert result.score == 0.0
|
|
|
|
|
|
def test_build_result_derives_correct_from_the_clamped_score():
|
|
result = grading.build_result({"correct": True, "score": 0.0})
|
|
assert result.is_correct is False
|
|
|
|
|
|
def test_build_result_marks_a_high_score_correct_despite_the_model():
|
|
result = grading.build_result({"correct": False, "score": 0.9})
|
|
assert result.is_correct is True
|
|
|
|
|
|
def test_build_result_falls_back_to_the_boolean_without_a_score():
|
|
assert grading.build_result({"correct": True}).score == 1.0
|
|
assert grading.build_result({"correct": False}).score == 0.0
|
|
|
|
|
|
def test_build_result_handles_an_unparsable_score():
|
|
assert grading.build_result({"correct": True, "score": "lots"}).score == 1.0
|
|
|
|
|
|
def test_build_result_clamps_the_confidence():
|
|
assert grading.build_result({"score": 1.0, "confidence": 9.0}).confidence == 1.0
|
|
assert grading.build_result({"score": 1.0, "confidence": -9.0}).confidence == 0.0
|
|
|
|
|
|
def test_build_result_truncates_and_strips_feedback():
|
|
parsed = {"score": 1.0, "feedback": "<b>nice</b> " + "x" * (QUIZ_FEEDBACK_MAX_CHARS + 50)}
|
|
result = grading.build_result(parsed)
|
|
assert "<b>" not in result.feedback
|
|
assert len(result.feedback) <= QUIZ_FEEDBACK_MAX_CHARS
|
|
|
|
|
|
def test_build_result_always_has_feedback():
|
|
assert grading.build_result({"score": 1.0}).feedback
|
|
|
|
|
|
def test_build_result_is_stamped_as_ai():
|
|
assert grading.build_result({"score": 1.0}).graded_by == "ai"
|
|
|
|
|
|
def test_build_prompt_carries_the_grading_context():
|
|
payload = json.loads(grading.build_prompt(QUESTION, "because the planner"))
|
|
assert payload["reference_answer"] == QUESTION["expected_answer"]
|
|
assert payload["grading_criteria"] == QUESTION["grading_criteria"]
|
|
assert payload["learner_answer"] == "because the planner"
|
|
|
|
|
|
def test_system_prompt_treats_the_answer_as_data():
|
|
assert "DATA" in grading.SYSTEM_PROMPT
|
|
assert "JSON" in grading.SYSTEM_PROMPT
|
|
|
|
|
|
def test_grade_free_text_falls_back_without_an_api_key():
|
|
result = grading.grade_free_text("", QUESTION, "the planner picks a bad index")
|
|
assert result.graded_by == "fallback"
|
|
assert 0.0 <= result.score <= 1.0
|
|
|
|
|
|
def test_grade_free_text_falls_back_when_the_gateway_raises(monkeypatch):
|
|
def boom(*args, **kwargs):
|
|
raise RuntimeError("gateway down")
|
|
|
|
monkeypatch.setattr(grading, "gateway_complete", boom)
|
|
result = grading.grade_free_text("key", QUESTION, "the planner picks a bad index")
|
|
assert result.graded_by == "fallback"
|
|
assert "unavailable" in result.feedback
|
|
|
|
|
|
def test_grade_free_text_falls_back_on_an_unreadable_reply(monkeypatch):
|
|
monkeypatch.setattr(grading, "gateway_complete", lambda *a, **k: ("not json", None))
|
|
result = grading.grade_free_text("key", QUESTION, "the planner picks a bad index")
|
|
assert result.graded_by == "fallback"
|
|
|
|
|
|
def test_grade_free_text_uses_a_valid_verdict(monkeypatch):
|
|
reply = json.dumps({"correct": True, "score": 0.75, "feedback": "Close.", "confidence": 0.9})
|
|
monkeypatch.setattr(grading, "gateway_complete", lambda *a, **k: (reply, None))
|
|
result = grading.grade_free_text("key", QUESTION, "the planner")
|
|
assert result.graded_by == "ai"
|
|
assert result.score == 0.75
|
|
assert result.is_correct is True
|
|
assert result.feedback == "Close."
|
|
|
|
|
|
def test_grade_free_text_passes_the_answering_key_through(monkeypatch):
|
|
seen = {}
|
|
|
|
def capture(api_key, system, text, timeout, model=None, bypass_preamble=False, app_reference=None):
|
|
seen["api_key"] = api_key
|
|
seen["bypass_preamble"] = bypass_preamble
|
|
return json.dumps({"score": 1.0}), None
|
|
|
|
monkeypatch.setattr(grading, "gateway_complete", capture)
|
|
grading.grade_free_text("member-key", QUESTION, "answer")
|
|
assert seen["api_key"] == "member-key"
|
|
|
|
|
|
def test_grade_free_text_bypasses_the_operator_preamble(monkeypatch):
|
|
seen = {}
|
|
|
|
def capture(api_key, system, text, timeout, model=None, bypass_preamble=False, app_reference=None):
|
|
seen["bypass_preamble"] = bypass_preamble
|
|
return json.dumps({"score": 1.0}), None
|
|
|
|
monkeypatch.setattr(grading, "gateway_complete", capture)
|
|
grading.grade_free_text("member-key", QUESTION, "answer")
|
|
assert seen["bypass_preamble"] is True
|
|
|
|
|
|
def test_the_fallback_score_is_the_deterministic_overlap():
|
|
expected = scoring.token_overlap_score(QUESTION["expected_answer"], "one bucket index")
|
|
result = grading.grade_free_text("", QUESTION, "one bucket index")
|
|
assert result.score == expected
|