forked from retoor/devplacepy
241 lines
7.4 KiB
Python
241 lines
7.4 KiB
Python
# retoor <retoor@molodetz.nl>
|
|||
|
|
|
||
|
|
import httpx
|
||
|
|
|
||
|
|
from devplacepy.services.openai_gateway import health, routing
|
||
|
|
from tests.conftest import run_async
|
||
|
|
|
||
|
|
|
||
|
|
class _FakeProbeResponse:
|
||
|
|
def __init__(self, status_code, payload=None, text=""):
|
||
|
|
self.status_code = status_code
|
||
|
|
self._payload = payload
|
||
|
|
self.text = text
|
||
|
|
|
||
|
|
def json(self):
|
||
|
|
if isinstance(self._payload, Exception):
|
||
|
|
raise self._payload
|
||
|
|
return self._payload
|
||
|
|
|
||
|
|
|
||
|
|
class _FakeProbeClient:
|
||
|
|
instances = []
|
||
|
|
|
||
|
|
def __init__(self, response=None, error=None):
|
||
|
|
self.response = response
|
||
|
|
self.error = error
|
||
|
|
self.requested_url = None
|
||
|
|
self.sent_body = None
|
||
|
|
self.headers = None
|
||
|
|
_FakeProbeClient.instances.append(self)
|
||
|
|
|
||
|
|
async def __aenter__(self):
|
||
|
|
return self
|
||
|
|
|
||
|
|
async def __aexit__(self, *exc_info):
|
||
|
|
return False
|
||
|
|
|
||
|
|
async def post(self, url, json=None, headers=None):
|
||
|
|
self.requested_url = url
|
||
|
|
self.sent_body = json
|
||
|
|
if self.error is not None:
|
||
|
|
raise self.error
|
||
|
|
return self.response
|
||
|
|
|
||
|
|
|
||
|
|
def _patch_probe(monkeypatch, response=None, error=None):
|
||
|
|
def _factory(**kwargs):
|
||
|
|
client = _FakeProbeClient(response=response, error=error)
|
||
|
|
client.headers = kwargs.get("headers")
|
||
|
|
return client
|
||
|
|
|
||
|
|
monkeypatch.setattr("devplacepy.stealth.stealth_async_client", _factory)
|
||
|
|
|
||
|
|
|
||
|
|
def _probe_provider(name="healthprov", base_url=None, protocol="chat_completions"):
|
||
|
|
routing.provider_store.set(
|
||
|
|
routing.ProviderIn(
|
||
|
|
name=name,
|
||
|
|
base_url=base_url or "https://probe.example/v1/chat/completions",
|
||
|
|
api_key="k",
|
||
|
|
protocol=protocol,
|
||
|
|
)
|
||
|
|
)
|
||
|
|
|
||
|
|
|
||
|
|
def _probe_route(source, kind="chat", target="vendor/probe"):
|
||
|
|
routing.model_store.set(
|
||
|
|
routing.ModelRouteIn(
|
||
|
|
source_model=source,
|
||
|
|
provider="healthprov",
|
||
|
|
target_model=target,
|
||
|
|
kind=kind,
|
||
|
|
)
|
||
|
|
)
|
||
|
|
|
||
|
|
|
||
|
|
def _cleanup(names):
|
||
|
|
for source in names:
|
||
|
|
routing.model_store.remove(source)
|
||
|
|
routing.provider_store.remove("healthprov")
|
||
|
|
|
||
|
|
|
||
|
|
def test_probe_chat_ok(local_db, monkeypatch):
|
||
|
|
_probe_provider()
|
||
|
|
_probe_route("probe-chat")
|
||
|
|
_patch_probe(
|
||
|
|
monkeypatch,
|
||
|
|
_FakeProbeResponse(200, {"choices": [{"message": {"content": "OK"}}]}),
|
||
|
|
)
|
||
|
|
try:
|
||
|
|
result = run_async(health.probe_route("probe-chat"))
|
||
|
|
assert result.ok
|
||
|
|
assert result.code == "ok"
|
||
|
|
client = _FakeProbeClient.instances[-1]
|
||
|
|
assert client.requested_url.endswith("/chat/completions")
|
||
|
|
assert client.sent_body["model"] == "vendor/probe"
|
||
|
|
assert client.sent_body["stream"] is False
|
||
|
|
finally:
|
||
|
|
_cleanup(["probe-chat"])
|
||
|
|
|
||
|
|
|
||
|
|
def test_probe_chat_non_200_records_code(local_db, monkeypatch):
|
||
|
|
_probe_provider()
|
||
|
|
_probe_route("probe-chat-bad")
|
||
|
|
_patch_probe(
|
||
|
|
monkeypatch,
|
||
|
|
_FakeProbeResponse(
|
||
|
|
403,
|
||
|
|
{"error": {"message": "denied", "code": "client_attestation_denied"}},
|
||
|
|
'{"error": {"message": "denied", "code": "client_attestation_denied"}}',
|
||
|
|
),
|
||
|
|
)
|
||
|
|
try:
|
||
|
|
result = run_async(health.probe_route("probe-chat-bad"))
|
||
|
|
assert not result.ok
|
||
|
|
assert result.code == "client_attestation_denied"
|
||
|
|
assert "denied" in result.message
|
||
|
|
finally:
|
||
|
|
_cleanup(["probe-chat-bad"])
|
||
|
|
|
||
|
|
|
||
|
|
def test_probe_chat_timeout(local_db, monkeypatch):
|
||
|
|
_probe_provider()
|
||
|
|
_probe_route("probe-chat-slow")
|
||
|
|
_patch_probe(monkeypatch, error=httpx.TimeoutException("slow"))
|
||
|
|
try:
|
||
|
|
result = run_async(health.probe_route("probe-chat-slow"))
|
||
|
|
assert not result.ok
|
||
|
|
assert result.code == "timeout"
|
||
|
|
finally:
|
||
|
|
_cleanup(["probe-chat-slow"])
|
||
|
|
|
||
|
|
|
||
|
|
def test_probe_chat_responses_protocol_transpiles(local_db, monkeypatch):
|
||
|
|
_probe_provider(
|
||
|
|
base_url="https://probe.example/v1/responses", protocol="responses"
|
||
|
|
)
|
||
|
|
_probe_route("probe-resp")
|
||
|
|
_patch_probe(
|
||
|
|
monkeypatch,
|
||
|
|
_FakeProbeResponse(
|
||
|
|
200,
|
||
|
|
{
|
||
|
|
"object": "response",
|
||
|
|
"status": "completed",
|
||
|
|
"output": [
|
||
|
|
{
|
||
|
|
"type": "message",
|
||
|
|
"content": [{"type": "output_text", "text": "OK"}],
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"usage": {"input_tokens": 1, "output_tokens": 1, "total_tokens": 2},
|
||
|
|
},
|
||
|
|
),
|
||
|
|
)
|
||
|
|
try:
|
||
|
|
result = run_async(health.probe_route("probe-resp"))
|
||
|
|
assert result.ok
|
||
|
|
client = _FakeProbeClient.instances[-1]
|
||
|
|
assert "input" in client.sent_body
|
||
|
|
assert client.sent_body["reasoning"] == {"effort": "minimal"}
|
||
|
|
finally:
|
||
|
|
_cleanup(["probe-resp"])
|
||
|
|
|
||
|
|
|
||
|
|
def test_probe_embed_and_image(local_db, monkeypatch):
|
||
|
|
_probe_provider()
|
||
|
|
_probe_route("probe-embed", kind="embed")
|
||
|
|
_patch_probe(
|
||
|
|
monkeypatch, _FakeProbeResponse(200, {"data": [{"embedding": [0.1]}]})
|
||
|
|
)
|
||
|
|
try:
|
||
|
|
result = run_async(health.probe_route("probe-embed"))
|
||
|
|
assert result.ok
|
||
|
|
client = _FakeProbeClient.instances[-1]
|
||
|
|
assert client.requested_url.endswith("/embeddings")
|
||
|
|
finally:
|
||
|
|
routing.model_store.remove("probe-embed")
|
||
|
|
_probe_route("probe-image", kind="image")
|
||
|
|
_patch_probe(
|
||
|
|
monkeypatch,
|
||
|
|
_FakeProbeResponse(200, {"data": [{"url": "https://img.example/h.png"}]}),
|
||
|
|
)
|
||
|
|
try:
|
||
|
|
result = run_async(health.probe_route("probe-image"))
|
||
|
|
assert result.ok
|
||
|
|
assert result.image_url == "https://img.example/h.png"
|
||
|
|
client = _FakeProbeClient.instances[-1]
|
||
|
|
assert client.sent_body["prompt"] == "a hedgehog and a bear"
|
||
|
|
finally:
|
||
|
|
_cleanup(["probe-image"])
|
||
|
|
|
||
|
|
|
||
|
|
def test_probe_unknown_route_returns_none(local_db):
|
||
|
|
assert run_async(health.probe_route("ghost-probe")) is None
|
||
|
|
|
||
|
|
|
||
|
|
def test_all_probes_every_route_one_by_one(local_db, monkeypatch):
|
||
|
|
_probe_provider()
|
||
|
|
_probe_route("probe-all-a")
|
||
|
|
_probe_route("probe-all-b")
|
||
|
|
seen = []
|
||
|
|
|
||
|
|
async def fake_probe(source):
|
||
|
|
seen.append(source)
|
||
|
|
ok = source != "probe-all-b"
|
||
|
|
return health.ProbeResult(ok, "" if ok else "http_500", "" if ok else "x", 1.0)
|
||
|
|
|
||
|
|
monkeypatch.setattr(health, "probe_route", fake_probe)
|
||
|
|
try:
|
||
|
|
tested, failed = run_async(health.test_all())
|
||
|
|
assert "probe-all-a" in seen and "probe-all-b" in seen
|
||
|
|
assert tested >= 2 and failed >= 1
|
||
|
|
assert routing.model_store.get("probe-all-a").health_status == "ok"
|
||
|
|
assert routing.model_store.get("probe-all-b").health_status == "bad"
|
||
|
|
finally:
|
||
|
|
_cleanup(["probe-all-a", "probe-all-b"])
|
||
|
|
|
||
|
|
|
||
|
|
def test_record_health_writes_and_clears(local_db):
|
||
|
|
_probe_provider()
|
||
|
|
_probe_route("probe-record")
|
||
|
|
try:
|
||
|
|
assert health.record_health(
|
||
|
|
"probe-record", health.ProbeResult(False, "http_500", "boom", 3.0)
|
||
|
|
)
|
||
|
|
route = routing.model_store.get("probe-record")
|
||
|
|
assert route.health_status == "bad"
|
||
|
|
assert route.health_code == "http_500"
|
||
|
|
assert route.health_message == "boom"
|
||
|
|
assert route.health_checked_at
|
||
|
|
assert health.record_health(
|
||
|
|
"probe-record", health.ProbeResult(True, "ok", "", 1.0)
|
||
|
|
)
|
||
|
|
route = routing.model_store.get("probe-record")
|
||
|
|
assert route.health_status == "ok"
|
||
|
|
assert route.health_code == ""
|
||
|
|
assert not health.record_health("ghost-record", health.ProbeResult(True, "ok", "", 1.0))
|
||
|
|
finally:
|
||
|
|
_cleanup(["probe-record"])
|