84 lines
3.1 KiB
Python
84 lines
3.1 KiB
Python
 
|
# retoor <retoor@molodetz.nl>
|
||
|
|
|
||
|
|
from devplacepy.services.openai_gateway import model_health
|
||
|
|
|
||
|
|
|
||
|
|
def setup_function():
|
||
|
|
model_health.reset()
|
||
|
|
|
||
|
|
|
||
|
|
def test_speed_reward_is_neutral_without_data():
|
||
|
|
assert model_health.speed_reward(None) == model_health.NEUTRAL_REWARD
|
||
|
|
assert model_health.speed_reward(0) == model_health.NEUTRAL_REWARD
|
||
|
|
|
||
|
|
|
||
|
|
def test_speed_reward_increases_with_throughput():
|
||
|
|
slow = model_health.speed_reward(5)
|
||
|
|
fast = model_health.speed_reward(50)
|
||
|
|
assert 0 < slow < fast < 1
|
||
|
|
|
||
|
|
|
||
|
|
def test_latency_reward_is_one_without_data():
|
||
|
|
assert model_health.latency_reward(None) == 1.0
|
||
|
|
assert model_health.latency_reward(0) == 1.0
|
||
|
|
|
||
|
|
|
||
|
|
def test_latency_reward_decreases_with_latency():
|
||
|
|
fast = model_health.latency_reward(500)
|
||
|
|
slow = model_health.latency_reward(20000)
|
||
|
|
assert 0 < slow < fast <= 1
|
||
|
|
|
||
|
|
|
||
|
|
def test_untested_model_has_neutral_weight():
|
||
|
|
health = model_health.ModelHealth()
|
||
|
|
assert health.weight() == 0.5
|
||
|
|
|
||
|
|
|
||
|
|
def test_record_outcome_success_improves_weight_over_failures():
|
||
|
|
model_health.record_outcome("openrouter", "fast-model", True, latency_ms=200, tokens_per_second=80)
|
||
|
|
model_health.record_outcome("openrouter", "slow-model", False)
|
||
|
|
fast_weight = model_health.snapshot_for("openrouter", "fast-model")["weight"]
|
||
|
|
slow_weight = model_health.snapshot_for("openrouter", "slow-model")["weight"]
|
||
|
|
assert fast_weight > 0.5
|
||
|
|
assert slow_weight < 0.5
|
||
|
|
|
||
|
|
|
||
|
|
def test_repeated_failures_open_the_circuit_for_display_only():
|
||
|
|
for _ in range(model_health.CIRCUIT_BREAKER_FAILURE_THRESHOLD):
|
||
|
|
model_health.record_outcome("deepseek", "flaky-model", False)
|
||
|
|
snapshot = model_health.snapshot_for("deepseek", "flaky-model")
|
||
|
|
assert snapshot["circuit_open"] is True
|
||
|
|
assert snapshot["consecutive_failures"] == model_health.CIRCUIT_BREAKER_FAILURE_THRESHOLD
|
||
|
|
|
||
|
|
|
||
|
|
def test_success_resets_consecutive_failures_and_circuit():
|
||
|
|
model_health.record_outcome("deepseek", "recovering-model", False)
|
||
|
|
model_health.record_outcome("deepseek", "recovering-model", False)
|
||
|
|
model_health.record_outcome("deepseek", "recovering-model", True, latency_ms=100, tokens_per_second=50)
|
||
|
|
snapshot = model_health.snapshot_for("deepseek", "recovering-model")
|
||
|
|
assert snapshot["consecutive_failures"] == 0
|
||
|
|
assert snapshot["circuit_open"] is False
|
||
|
|
|
||
|
|
|
||
|
|
def test_snapshot_for_unknown_model_is_none():
|
||
|
|
assert model_health.snapshot_for("openrouter", "never-seen") is None
|
||
|
|
|
||
|
|
|
||
|
|
def test_apply_history_accumulates_without_touching_circuit_state():
|
||
|
|
model_health.apply_history("openrouter", "seeded-model", success_count=10, failure_count=2, total_reward=6.0)
|
||
|
|
snapshot = model_health.snapshot_for("openrouter", "seeded-model")
|
||
|
|
assert snapshot["success_count"] == 10
|
||
|
|
assert snapshot["failure_count"] == 2
|
||
|
|
assert snapshot["circuit_open"] is False
|
||
|
|
|
||
|
|
|
||
|
|
def test_record_outcome_requires_a_model_name():
|
||
|
|
model_health.record_outcome("openrouter", "", True, latency_ms=100)
|
||
|
|
assert model_health.snapshot_all() == {}
|
||
|
|
|
||
|
|
|
||
|
|
def test_snapshot_all_keys_are_provider_colon_model():
|
||
|
|
model_health.record_outcome("groq", "llama-3", True)
|
||
|
|
keys = model_health.snapshot_all().keys()
|
||
|
|
assert "groq:llama-3" in keys
|