forked from retoor/devplacepy
4167 lines
142 KiB
Python
4167 lines
142 KiB
Python
# retoor <retoor@molodetz.nl>
|
|
|
|
import json
|
|
from starlette.requests import Request
|
|
from tests.conftest import BASE_URL, run_async
|
|
from devplacepy.database import get_table, set_setting
|
|
from devplacepy.utils import generate_uid
|
|
import devplacepy.services.openai_gateway.gateway as gwmod
|
|
from devplacepy.services.openai_gateway import GatewayService
|
|
from devplacepy.services.openai_gateway import auth_throttle
|
|
class FakeResp_openai_gateway:
|
|
def __init__(
|
|
self,
|
|
status=200,
|
|
payload=None,
|
|
ctype="application/json",
|
|
content=b"",
|
|
extra_headers=None,
|
|
):
|
|
self.status_code = status
|
|
self._payload = payload
|
|
self.text = json.dumps(payload) if payload is not None else ""
|
|
self.headers = {"content-type": ctype, **(extra_headers or {})}
|
|
self.content = content
|
|
|
|
def json(self):
|
|
if self._payload is None:
|
|
raise ValueError("no json")
|
|
return self._payload
|
|
|
|
async def aread(self):
|
|
return self.content or self.text.encode()
|
|
|
|
async def aclose(self):
|
|
pass
|
|
|
|
async def aiter_lines(self):
|
|
payload = self._payload or {}
|
|
chunk_id = payload.get("id", "x")
|
|
model = payload.get("model", "")
|
|
try:
|
|
content = payload["choices"][0]["message"].get("content") or ""
|
|
except (KeyError, IndexError, TypeError):
|
|
content = ""
|
|
try:
|
|
reasoning = payload["choices"][0]["message"].get("reasoning") or ""
|
|
except (KeyError, IndexError, TypeError):
|
|
reasoning = ""
|
|
|
|
def frame(delta=None, finish=None, usage=None):
|
|
body = {"id": chunk_id, "object": "chat.completion.chunk", "model": model}
|
|
if usage is not None:
|
|
body["choices"] = []
|
|
body["usage"] = usage
|
|
else:
|
|
body["choices"] = [{"index": 0, "delta": delta or {}, "finish_reason": finish}]
|
|
return f"data: {json.dumps(body)}"
|
|
|
|
yield frame({"role": "assistant"})
|
|
for i in range(0, len(reasoning), 5):
|
|
yield frame({"reasoning": reasoning[i : i + 5]})
|
|
for i in range(0, len(content), 5):
|
|
yield frame({"content": content[i : i + 5]})
|
|
yield frame({}, finish="stop")
|
|
if payload.get("usage"):
|
|
yield frame(usage=payload["usage"])
|
|
yield "data: [DONE]"
|
|
class FakeRequest:
|
|
def __init__(self, method, url, json_body, headers=None):
|
|
self.method = method
|
|
self.url = url
|
|
self.json_body = json_body
|
|
self.headers = headers or {}
|
|
self.extensions = {}
|
|
class FakeClient_openai_gateway:
|
|
def __init__(self, *a, **k):
|
|
self.calls = []
|
|
|
|
def build_request(self, method, url, headers=None, json=None, content=None):
|
|
return FakeRequest(method, url, json)
|
|
|
|
async def send(self, request, stream=False):
|
|
self.calls.append((request.url, request.json_body))
|
|
body = request.json_body or {}
|
|
return FakeResp_openai_gateway(
|
|
payload={
|
|
"id": "x",
|
|
"model": body.get("model"),
|
|
"choices": [{"message": {"content": "hi there"}}],
|
|
}
|
|
)
|
|
|
|
async def post(self, url, headers=None, json=None, timeout=None):
|
|
self.calls.append((url, json))
|
|
return FakeResp_openai_gateway(
|
|
payload={
|
|
"id": "x",
|
|
"model": json.get("model"),
|
|
"choices": [{"message": {"content": "hi there"}}],
|
|
}
|
|
)
|
|
|
|
async def request(self, method, url, headers=None, content=None):
|
|
return FakeResp_openai_gateway(payload={"ok": True})
|
|
|
|
async def aclose(self):
|
|
pass
|
|
def _make_request_openai_gateway(headers=None, cookies=None):
|
|
headers = headers or {}
|
|
raw = [(k.lower().encode(), v.encode()) for k, v in headers.items()]
|
|
if cookies:
|
|
raw.append(
|
|
(b"cookie", "; ".join(f"{k}={v}" for k, v in cookies.items()).encode())
|
|
)
|
|
return Request(
|
|
{
|
|
"type": "http",
|
|
"method": "POST",
|
|
"path": "/openai/v1/chat/completions",
|
|
"query_string": b"",
|
|
"headers": raw,
|
|
"state": {},
|
|
}
|
|
)
|
|
def _make_admin_openai_gateway(role="Admin"):
|
|
username = f"gw_{generate_uid()[:8]}"
|
|
api_key = generate_uid()
|
|
get_table("users").insert(
|
|
{
|
|
"uid": generate_uid(),
|
|
"username": username,
|
|
"terms_version": "1",
|
|
"email": f"{username}@t.dev",
|
|
"api_key": api_key,
|
|
"role": role,
|
|
"is_active": True,
|
|
}
|
|
)
|
|
return username, api_key
|
|
def _config(page, **fields):
|
|
page.request.post(f"{BASE_URL}/admin/services/openai/config", form=fields)
|
|
|
|
|
|
def test_model_is_forced(local_db, monkeypatch):
|
|
monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeClient_openai_gateway)
|
|
svc = GatewayService()
|
|
cfg = svc.effective_config()
|
|
cfg["gateway_force_model"] = True
|
|
cfg["gateway_model"] = "deepseek-chat"
|
|
rt = svc.runtime()
|
|
run_async(
|
|
rt.handle_chat(
|
|
{"model": "gpt-4", "messages": [{"role": "user", "content": "hi"}]},
|
|
cfg,
|
|
("guest", "test"),
|
|
"test",
|
|
"default",
|
|
)
|
|
)
|
|
assert rt._client.calls[-1][1]["model"] == "deepseek-chat"
|
|
|
|
|
|
def test_model_route_overrides_upstream(local_db, monkeypatch):
|
|
from devplacepy.services.openai_gateway import routing
|
|
|
|
monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeClient_openai_gateway)
|
|
routing.provider_store.set(
|
|
routing.ProviderIn(
|
|
name="gwroute",
|
|
base_url="https://routed.example/v1/chat/completions",
|
|
api_key="routed-key",
|
|
)
|
|
)
|
|
routing.model_store.set(
|
|
routing.ModelRouteIn(
|
|
source_model="custom-pro",
|
|
provider="gwroute",
|
|
target_model="vendor/pro",
|
|
kind="chat",
|
|
price_output_per_m=9.0,
|
|
)
|
|
)
|
|
try:
|
|
svc = GatewayService()
|
|
cfg = svc.effective_config()
|
|
cfg["gateway_vision_enabled"] = False
|
|
rt = svc.runtime()
|
|
run_async(
|
|
rt.handle_chat(
|
|
{"model": "custom-pro", "messages": [{"role": "user", "content": "hi"}]},
|
|
cfg,
|
|
("guest", "test"),
|
|
"test",
|
|
"default",
|
|
)
|
|
)
|
|
url, body = rt._client.calls[-1]
|
|
assert str(url) == "https://routed.example/v1/chat/completions"
|
|
assert body["model"] == "vendor/pro"
|
|
finally:
|
|
routing.model_store.remove("custom-pro")
|
|
routing.provider_store.remove("gwroute")
|
|
|
|
|
|
def test_vision_rewrites_image_to_text(local_db, monkeypatch):
|
|
monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeClient_openai_gateway)
|
|
svc = GatewayService()
|
|
cfg = svc.effective_config()
|
|
cfg["gateway_vision_enabled"] = True
|
|
cfg["gateway_vision_key"] = "" # no key -> placeholder text, still rewrites to str
|
|
rt = svc.runtime()
|
|
msgs = [
|
|
{
|
|
"role": "user",
|
|
"content": [
|
|
{"type": "text", "text": "what is this"},
|
|
{"type": "image_url", "image_url": {"url": "http://x/y.png"}},
|
|
],
|
|
}
|
|
]
|
|
run_async(rt.handle_chat({"messages": msgs}, cfg, ("guest", "test"), "test", "default"))
|
|
sent = rt._client.calls[-1][1]["messages"][0]["content"]
|
|
assert isinstance(sent, str) and "vision" in sent.lower()
|
|
|
|
|
|
def test_streaming_emits_sse(local_db, monkeypatch):
|
|
monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeClient_openai_gateway)
|
|
svc = GatewayService()
|
|
cfg = svc.effective_config()
|
|
rt = svc.runtime()
|
|
resp = run_async(
|
|
rt.handle_chat(
|
|
{"messages": [{"role": "user", "content": "hi"}], "stream": True},
|
|
cfg,
|
|
("guest", "test"),
|
|
"test",
|
|
"default",
|
|
)
|
|
)
|
|
|
|
async def drain():
|
|
out = []
|
|
async for chunk in resp.body_iterator:
|
|
out.append(chunk if isinstance(chunk, str) else chunk.decode())
|
|
return "".join(out)
|
|
|
|
body = run_async(drain())
|
|
assert "chat.completion.chunk" in body
|
|
assert "[DONE]" in body
|
|
|
|
|
|
def test_authorize_static_access_key(local_db):
|
|
set_setting("gateway_require_auth", "1")
|
|
set_setting("gateway_access_key", "topsecret")
|
|
try:
|
|
svc = GatewayService()
|
|
assert svc.authorize(_make_request_openai_gateway(headers={"X-API-KEY": "topsecret"})) is True
|
|
assert svc.authorize(_make_request_openai_gateway(headers={"X-API-KEY": "wrong"})) is False
|
|
finally:
|
|
set_setting("gateway_access_key", "")
|
|
set_setting("gateway_require_auth", "1")
|
|
auth_throttle.clear()
|
|
|
|
|
|
def test_authorize_admin_and_user_toggles(local_db):
|
|
set_setting("gateway_require_auth", "1")
|
|
set_setting("gateway_access_key", "")
|
|
set_setting("gateway_allow_admins", "1")
|
|
set_setting("gateway_allow_users", "0")
|
|
try:
|
|
_, admin_key = _make_admin_openai_gateway(role="Admin")
|
|
_, member_key = _make_admin_openai_gateway(role="Member")
|
|
svc = GatewayService()
|
|
|
|
assert (
|
|
svc.authorize(_make_request_openai_gateway(headers={"Authorization": f"Bearer {admin_key}"}))
|
|
is True
|
|
)
|
|
assert svc.authorize(_make_request_openai_gateway(headers={"X-API-KEY": member_key})) is False
|
|
|
|
set_setting("gateway_allow_users", "1")
|
|
assert svc.authorize(_make_request_openai_gateway(headers={"X-API-KEY": member_key})) is True
|
|
finally:
|
|
set_setting("gateway_allow_admins", "1")
|
|
set_setting("gateway_allow_users", "1")
|
|
set_setting("gateway_require_auth", "1")
|
|
set_setting("gateway_access_key", "")
|
|
auth_throttle.clear()
|
|
|
|
|
|
def test_authorize_require_auth_off_is_open(local_db):
|
|
set_setting("gateway_require_auth", "0")
|
|
try:
|
|
svc = GatewayService()
|
|
assert svc.authorize(_make_request_openai_gateway()) is True
|
|
finally:
|
|
set_setting("gateway_require_auth", "1")
|
|
|
|
|
|
def test_authorize_throttles_repeated_failed_auth_from_one_ip(local_db):
|
|
import pytest
|
|
from fastapi import HTTPException
|
|
|
|
set_setting("gateway_require_auth", "1")
|
|
set_setting("gateway_access_key", "")
|
|
set_setting("gateway_auth_throttle_max_failures", "3")
|
|
set_setting("gateway_auth_throttle_window_seconds", "60")
|
|
auth_throttle.clear()
|
|
try:
|
|
svc = GatewayService()
|
|
ip = "203.0.113.10"
|
|
for _ in range(3):
|
|
assert (
|
|
svc.authorize(_make_request_openai_gateway(headers={"X-Real-IP": ip}))
|
|
is False
|
|
)
|
|
with pytest.raises(HTTPException) as excinfo:
|
|
svc.authorize(_make_request_openai_gateway(headers={"X-Real-IP": ip}))
|
|
assert excinfo.value.status_code == 429
|
|
assert "Retry-After" in excinfo.value.headers
|
|
finally:
|
|
set_setting("gateway_auth_throttle_max_failures", "10")
|
|
set_setting("gateway_auth_throttle_window_seconds", "60")
|
|
set_setting("gateway_require_auth", "1")
|
|
auth_throttle.clear()
|
|
|
|
|
|
def test_authorize_throttle_does_not_affect_other_ips(local_db):
|
|
set_setting("gateway_require_auth", "1")
|
|
set_setting("gateway_access_key", "")
|
|
set_setting("gateway_auth_throttle_max_failures", "3")
|
|
set_setting("gateway_auth_throttle_window_seconds", "60")
|
|
auth_throttle.clear()
|
|
try:
|
|
svc = GatewayService()
|
|
throttled_ip = "203.0.113.20"
|
|
other_ip = "203.0.113.21"
|
|
for _ in range(3):
|
|
svc.authorize(_make_request_openai_gateway(headers={"X-Real-IP": throttled_ip}))
|
|
assert auth_throttle.is_throttled(throttled_ip, 3, 60) is True
|
|
assert (
|
|
svc.authorize(_make_request_openai_gateway(headers={"X-Real-IP": other_ip}))
|
|
is False
|
|
)
|
|
assert auth_throttle.is_throttled(other_ip, 3, 60) is False
|
|
finally:
|
|
set_setting("gateway_auth_throttle_max_failures", "10")
|
|
set_setting("gateway_auth_throttle_window_seconds", "60")
|
|
set_setting("gateway_require_auth", "1")
|
|
auth_throttle.clear()
|
|
|
|
|
|
def test_authorize_valid_credentials_succeed_from_throttled_ip(local_db):
|
|
set_setting("gateway_require_auth", "1")
|
|
set_setting("gateway_access_key", "")
|
|
set_setting("gateway_allow_users", "1")
|
|
set_setting("gateway_auth_throttle_max_failures", "3")
|
|
set_setting("gateway_auth_throttle_window_seconds", "60")
|
|
auth_throttle.clear()
|
|
try:
|
|
svc = GatewayService()
|
|
ip = "203.0.113.30"
|
|
_, member_key = _make_admin_openai_gateway(role="Member")
|
|
for _ in range(3):
|
|
svc.authorize(_make_request_openai_gateway(headers={"X-Real-IP": ip}))
|
|
assert auth_throttle.is_throttled(ip, 3, 60) is True
|
|
assert (
|
|
svc.authorize(
|
|
_make_request_openai_gateway(
|
|
headers={"X-Real-IP": ip, "X-API-KEY": member_key}
|
|
)
|
|
)
|
|
is True
|
|
)
|
|
set_setting("gateway_access_key", "topsecret")
|
|
assert (
|
|
svc.authorize(
|
|
_make_request_openai_gateway(
|
|
headers={"X-Real-IP": ip, "X-API-KEY": "topsecret"}
|
|
)
|
|
)
|
|
is True
|
|
)
|
|
finally:
|
|
set_setting("gateway_access_key", "")
|
|
set_setting("gateway_auth_throttle_max_failures", "10")
|
|
set_setting("gateway_auth_throttle_window_seconds", "60")
|
|
set_setting("gateway_require_auth", "1")
|
|
auth_throttle.clear()
|
|
|
|
|
|
def test_authorize_throttle_resets_after_window(local_db, monkeypatch):
|
|
set_setting("gateway_require_auth", "1")
|
|
set_setting("gateway_access_key", "")
|
|
set_setting("gateway_auth_throttle_max_failures", "3")
|
|
set_setting("gateway_auth_throttle_window_seconds", "60")
|
|
auth_throttle.clear()
|
|
try:
|
|
now = [2000.0]
|
|
monkeypatch.setattr(auth_throttle.time, "time", lambda: now[0])
|
|
svc = GatewayService()
|
|
ip = "203.0.113.40"
|
|
for _ in range(3):
|
|
svc.authorize(_make_request_openai_gateway(headers={"X-Real-IP": ip}))
|
|
assert auth_throttle.is_throttled(ip, 3, 60) is True
|
|
now[0] += 61
|
|
assert (
|
|
svc.authorize(_make_request_openai_gateway(headers={"X-Real-IP": ip}))
|
|
is False
|
|
)
|
|
finally:
|
|
set_setting("gateway_auth_throttle_max_failures", "10")
|
|
set_setting("gateway_auth_throttle_window_seconds", "60")
|
|
set_setting("gateway_require_auth", "1")
|
|
auth_throttle.clear()
|
|
|
|
|
|
class FakeEmbedClient_openai_gateway:
|
|
def __init__(self, *a, **k):
|
|
self.calls = []
|
|
|
|
def build_request(self, method, url, headers=None, json=None, content=None):
|
|
return FakeRequest(method, url, json)
|
|
|
|
async def send(self, request, stream=False):
|
|
self.calls.append((request.url, request.json_body))
|
|
body = request.json_body or {}
|
|
return FakeResp_openai_gateway(
|
|
payload={
|
|
"object": "list",
|
|
"model": body.get("model"),
|
|
"data": [{"object": "embedding", "index": 0, "embedding": [0.1, 0.2]}],
|
|
"usage": {"prompt_tokens": 5, "total_tokens": 5},
|
|
}
|
|
)
|
|
|
|
async def aclose(self):
|
|
pass
|
|
|
|
|
|
def test_embeddings_remaps_alias_to_configured_model(local_db, monkeypatch):
|
|
monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeEmbedClient_openai_gateway)
|
|
monkeypatch.setenv("OPENROUTER_API_KEY", "test-embed-key")
|
|
svc = GatewayService()
|
|
cfg = svc.effective_config()
|
|
cfg["gateway_force_model"] = False
|
|
cfg["gateway_embed_enabled"] = True
|
|
cfg["gateway_embed_model"] = "qwen/qwen3-embedding-8b"
|
|
rt = svc.runtime()
|
|
run_async(
|
|
rt.handle_embeddings(
|
|
{"model": "molodetz~embed", "input": "hello"},
|
|
cfg,
|
|
("guest", "test"),
|
|
"test",
|
|
"default",
|
|
)
|
|
)
|
|
assert rt._client.calls[-1][1]["model"] == "qwen/qwen3-embedding-8b"
|
|
|
|
|
|
def test_embeddings_success_records_ledger(local_db, monkeypatch):
|
|
monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeEmbedClient_openai_gateway)
|
|
monkeypatch.setenv("OPENROUTER_API_KEY", "test-embed-key")
|
|
svc = GatewayService()
|
|
cfg = svc.effective_config()
|
|
cfg["gateway_force_model"] = True
|
|
cfg["gateway_embed_enabled"] = True
|
|
cfg["gateway_embed_model"] = "qwen/qwen3-embedding-8b"
|
|
rt = svc.runtime()
|
|
before = rt.embed_calls
|
|
resp = run_async(
|
|
rt.handle_embeddings(
|
|
{"model": "molodetz~embed", "input": "hello"},
|
|
cfg,
|
|
("guest", "ledger_probe"),
|
|
"test",
|
|
"default",
|
|
)
|
|
)
|
|
assert resp.status_code == 200
|
|
payload = json.loads(bytes(resp.body).decode())
|
|
assert payload["data"][0]["embedding"] == [0.1, 0.2]
|
|
assert rt.embed_calls == before + 1
|
|
row = get_table("gateway_usage_ledger").find_one(owner_id="ledger_probe")
|
|
assert row is not None
|
|
assert row["backend"] == "embed"
|
|
assert row["endpoint"] == "embeddings"
|
|
assert row["requested_model"] == "molodetz~embed"
|
|
assert row["model"] == "qwen/qwen3-embedding-8b"
|
|
assert row["success"] == 1
|
|
|
|
|
|
def test_embeddings_disabled_returns_503(local_db, monkeypatch):
|
|
monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeEmbedClient_openai_gateway)
|
|
svc = GatewayService()
|
|
cfg = svc.effective_config()
|
|
cfg["gateway_embed_enabled"] = False
|
|
rt = svc.runtime()
|
|
resp = run_async(
|
|
rt.handle_embeddings(
|
|
{"input": "hello"}, cfg, ("guest", "test"), "test", "default"
|
|
)
|
|
)
|
|
assert resp.status_code == 503
|
|
|
|
|
|
def test_embeddings_missing_key_returns_clean_error_not_unauthenticated_call(
|
|
local_db, monkeypatch
|
|
):
|
|
monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeEmbedClient_openai_gateway)
|
|
monkeypatch.delenv("OPENROUTER_API_KEY", raising=False)
|
|
monkeypatch.delenv("DEEPSEEK_API_KEY", raising=False)
|
|
set_setting("gateway_api_key", "")
|
|
set_setting("gateway_vision_key", "")
|
|
set_setting("gateway_embed_key", "")
|
|
try:
|
|
svc = GatewayService()
|
|
cfg = svc.effective_config()
|
|
assert cfg["gateway_embed_key"] == ""
|
|
cfg["gateway_embed_enabled"] = True
|
|
rt = svc.runtime()
|
|
resp = run_async(
|
|
rt.handle_embeddings(
|
|
{"input": "hello"}, cfg, ("guest", "no_key_probe"), "test", "default"
|
|
)
|
|
)
|
|
assert resp.status_code == 503
|
|
payload = json.loads(bytes(resp.body).decode())
|
|
assert payload["error"]["type"] == "embeddings_not_configured"
|
|
assert rt._client is None
|
|
finally:
|
|
set_setting("gateway_api_key", "")
|
|
set_setting("gateway_vision_key", "")
|
|
set_setting("gateway_embed_key", "")
|
|
|
|
|
|
class FakeImageClient_openai_gateway:
|
|
def __init__(self, *a, **k):
|
|
self.calls = []
|
|
|
|
def build_request(self, method, url, headers=None, json=None, content=None):
|
|
return FakeRequest(method, url, json)
|
|
|
|
async def send(self, request, stream=False):
|
|
self.calls.append((request.url, request.json_body))
|
|
body = request.json_body or {}
|
|
return FakeResp_openai_gateway(
|
|
payload={
|
|
"created": 1,
|
|
"model": body.get("model"),
|
|
"data": [{"b64_json": "aGVsbG8="}],
|
|
"usage": {"cost": 0.05},
|
|
}
|
|
)
|
|
|
|
async def aclose(self):
|
|
pass
|
|
|
|
|
|
def test_images_remaps_alias_to_configured_model(local_db, monkeypatch):
|
|
monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeImageClient_openai_gateway)
|
|
svc = GatewayService()
|
|
cfg = svc.effective_config()
|
|
cfg["gateway_force_model"] = False
|
|
cfg["gateway_image_enabled"] = True
|
|
cfg["gateway_image_model"] = "black-forest-labs/flux.2-pro"
|
|
rt = svc.runtime()
|
|
run_async(
|
|
rt.handle_images(
|
|
{
|
|
"model": "molodetz-img-small",
|
|
"prompt": "award emblem",
|
|
"size": "512x512",
|
|
},
|
|
cfg,
|
|
("guest", "test"),
|
|
"test",
|
|
"default",
|
|
)
|
|
)
|
|
assert rt._client.calls[-1][1]["model"] == "black-forest-labs/flux.2-pro"
|
|
|
|
|
|
def test_image_route_overrides_upstream(local_db, monkeypatch):
|
|
from devplacepy.services.openai_gateway import routing
|
|
|
|
monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeImageClient_openai_gateway)
|
|
routing.provider_store.set(
|
|
routing.ProviderIn(
|
|
name="gwimg",
|
|
base_url="https://routed.example/v1/chat/completions",
|
|
api_key="img-key",
|
|
)
|
|
)
|
|
routing.model_store.set(
|
|
routing.ModelRouteIn(
|
|
source_model="custom-img",
|
|
provider="gwimg",
|
|
target_model="vendor/flux-pro",
|
|
kind="image",
|
|
price_input_per_m=0.06,
|
|
)
|
|
)
|
|
try:
|
|
svc = GatewayService()
|
|
cfg = svc.effective_config()
|
|
rt = svc.runtime()
|
|
run_async(
|
|
rt.handle_images(
|
|
{
|
|
"model": "custom-img",
|
|
"prompt": "trophy",
|
|
"response_format": "b64_json",
|
|
},
|
|
cfg,
|
|
("guest", "test"),
|
|
"test",
|
|
"default",
|
|
)
|
|
)
|
|
url, body = rt._client.calls[-1]
|
|
assert str(url) == "https://routed.example/v1/images"
|
|
assert body["model"] == "vendor/flux-pro"
|
|
finally:
|
|
routing.model_store.remove("custom-img")
|
|
routing.provider_store.remove("gwimg")
|
|
|
|
|
|
def test_images_success_records_ledger(local_db, monkeypatch):
|
|
monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeImageClient_openai_gateway)
|
|
svc = GatewayService()
|
|
cfg = svc.effective_config()
|
|
cfg["gateway_image_enabled"] = True
|
|
cfg["gateway_image_model"] = "black-forest-labs/flux.2-pro"
|
|
rt = svc.runtime()
|
|
before = rt.image_calls
|
|
resp = run_async(
|
|
rt.handle_images(
|
|
{"model": "molodetz-img-small", "prompt": "badge"},
|
|
cfg,
|
|
("guest", "img_ledger"),
|
|
"test",
|
|
"default",
|
|
)
|
|
)
|
|
assert resp.status_code == 200
|
|
payload = json.loads(bytes(resp.body).decode())
|
|
assert payload["data"][0]["b64_json"] == "aGVsbG8="
|
|
assert rt.image_calls == before + 1
|
|
row = get_table("gateway_usage_ledger").find_one(owner_id="img_ledger")
|
|
assert row is not None
|
|
assert row["backend"] == "image"
|
|
assert row["endpoint"] == "images/generations"
|
|
assert row["requested_model"] == "molodetz-img-small"
|
|
assert row["model"] == "black-forest-labs/flux.2-pro"
|
|
assert row["success"] == 1
|
|
assert float(row["cost_usd"]) == 0.05
|
|
|
|
|
|
def test_images_disabled_returns_503(local_db, monkeypatch):
|
|
monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeImageClient_openai_gateway)
|
|
svc = GatewayService()
|
|
cfg = svc.effective_config()
|
|
cfg["gateway_image_enabled"] = False
|
|
rt = svc.runtime()
|
|
resp = run_async(
|
|
rt.handle_images(
|
|
{"prompt": "badge"}, cfg, ("guest", "test"), "test", "default"
|
|
)
|
|
)
|
|
assert resp.status_code == 503
|
|
|
|
|
|
def test_compute_cost_embed_branch():
|
|
from devplacepy.services.openai_gateway.usage import (
|
|
Pricing,
|
|
compute_cost,
|
|
normalize_usage,
|
|
)
|
|
|
|
pricing = Pricing(
|
|
chat_cache_hit_per_m=0.0,
|
|
chat_cache_miss_per_m=0.0,
|
|
chat_output_per_m=0.0,
|
|
vision_input_per_m=0.0,
|
|
vision_output_per_m=0.0,
|
|
embed_input_per_m=0.01,
|
|
)
|
|
usage = {"prompt_tokens": 1_000_000, "total_tokens": 1_000_000}
|
|
norm = normalize_usage(usage)
|
|
total, input_cost, output_cost, native = compute_cost(
|
|
usage, norm, pricing, "embed"
|
|
)
|
|
assert native is False
|
|
assert output_cost == 0.0
|
|
assert abs(total - 0.01) < 1e-9
|
|
assert abs(input_cost - 0.01) < 1e-9
|
|
|
|
|
|
def test_app_reference_stored_in_ledger_for_chat(local_db, monkeypatch):
|
|
monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeClient_openai_gateway)
|
|
svc = GatewayService()
|
|
cfg = svc.effective_config()
|
|
cfg["gateway_force_model"] = True
|
|
cfg["gateway_model"] = "deepseek-chat"
|
|
rt = svc.runtime()
|
|
run_async(
|
|
rt.handle_chat(
|
|
{"messages": [{"role": "user", "content": "ref_test"}]},
|
|
cfg,
|
|
("guest", "ref_probe"),
|
|
"test-ua",
|
|
"my-custom-app",
|
|
)
|
|
)
|
|
row = get_table("gateway_usage_ledger").find_one(owner_id="ref_probe")
|
|
assert row is not None
|
|
assert row["app_reference"] == "my-custom-app"
|
|
assert row["backend"] == "chat"
|
|
|
|
|
|
def test_app_reference_defaults_when_not_passed(local_db, monkeypatch):
|
|
monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeClient_openai_gateway)
|
|
svc = GatewayService()
|
|
cfg = svc.effective_config()
|
|
cfg["gateway_force_model"] = True
|
|
cfg["gateway_model"] = "deepseek-chat"
|
|
rt = svc.runtime()
|
|
run_async(
|
|
rt.handle_chat(
|
|
{"messages": [{"role": "user", "content": "default_test"}]},
|
|
cfg,
|
|
("guest", "default_probe"),
|
|
"test-ua",
|
|
"default",
|
|
)
|
|
)
|
|
row = get_table("gateway_usage_ledger").find_one(owner_id="default_probe")
|
|
assert row is not None
|
|
assert row["app_reference"] == "default"
|
|
|
|
|
|
def test_app_reference_stored_in_ledger_for_embeddings(local_db, monkeypatch):
|
|
monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeEmbedClient_openai_gateway)
|
|
monkeypatch.setenv("OPENROUTER_API_KEY", "test-embed-key")
|
|
svc = GatewayService()
|
|
cfg = svc.effective_config()
|
|
cfg["gateway_embed_enabled"] = True
|
|
cfg["gateway_embed_model"] = "qwen/qwen3-embedding-8b"
|
|
rt = svc.runtime()
|
|
run_async(
|
|
rt.handle_embeddings(
|
|
{"input": "hello"},
|
|
cfg,
|
|
("guest", "embed_ref"),
|
|
"test-ua",
|
|
"devplace-embed-test-v-1-0-0",
|
|
)
|
|
)
|
|
row = get_table("gateway_usage_ledger").find_one(owner_id="embed_ref")
|
|
assert row is not None
|
|
assert row["app_reference"] == "devplace-embed-test-v-1-0-0"
|
|
assert row["backend"] == "embed"
|
|
|
|
|
|
def test_app_reference_stored_in_ledger_for_images(local_db, monkeypatch):
|
|
monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeImageClient_openai_gateway)
|
|
svc = GatewayService()
|
|
cfg = svc.effective_config()
|
|
cfg["gateway_image_enabled"] = True
|
|
cfg["gateway_image_model"] = "black-forest-labs/flux.2-pro"
|
|
rt = svc.runtime()
|
|
run_async(
|
|
rt.handle_images(
|
|
{"prompt": "ref badge"},
|
|
cfg,
|
|
("guest", "img_ref"),
|
|
"test-ua",
|
|
"devplace-image-test-v-1-0-0",
|
|
)
|
|
)
|
|
row = get_table("gateway_usage_ledger").find_one(owner_id="img_ref")
|
|
assert row is not None
|
|
assert row["app_reference"] == "devplace-image-test-v-1-0-0"
|
|
assert row["backend"] == "image"
|
|
|
|
|
|
def test_app_reference_column_exists(local_db):
|
|
db = get_table("gateway_usage_ledger").db
|
|
if "gateway_usage_ledger" not in db.tables:
|
|
return
|
|
rows = list(db.query("PRAGMA table_info('gateway_usage_ledger')"))
|
|
column_names = [r["name"] for r in rows]
|
|
assert "app_reference" in column_names
|
|
|
|
|
|
def test_chat_unknown_model_falls_back_to_default(local_db, monkeypatch):
|
|
monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeClient_openai_gateway)
|
|
svc = GatewayService()
|
|
cfg = svc.effective_config()
|
|
cfg["gateway_force_model"] = False
|
|
cfg["gateway_model"] = "deepseek-v4-flash"
|
|
rt = svc.runtime()
|
|
run_async(
|
|
rt.handle_chat(
|
|
{"model": "nonexistent-model-v99", "messages": [{"role": "user", "content": "hi"}]},
|
|
cfg,
|
|
("guest", "fallback_test_chat"),
|
|
"test",
|
|
"default",
|
|
)
|
|
)
|
|
assert rt._client.calls[-1][1]["model"] == "deepseek-v4-flash"
|
|
|
|
|
|
class FakeFallbackClient_openai_gateway:
|
|
def __init__(self, *a, **k):
|
|
self.calls = []
|
|
|
|
def build_request(self, method, url, headers=None, json=None, content=None):
|
|
return FakeRequest(method, url, json)
|
|
|
|
async def send(self, request, stream=False):
|
|
self.calls.append((request.url, request.json_body))
|
|
body = request.json_body or {}
|
|
model = body.get("model")
|
|
if model == "primary-target":
|
|
return FakeResp_openai_gateway(status=500, payload={"error": "boom"})
|
|
return FakeResp_openai_gateway(
|
|
payload={
|
|
"id": "x",
|
|
"model": model,
|
|
"choices": [{"message": {"content": "fallback ok"}}],
|
|
}
|
|
)
|
|
|
|
async def aclose(self):
|
|
pass
|
|
|
|
|
|
class FakeAlwaysFailClient_openai_gateway:
|
|
def __init__(self, *a, **k):
|
|
self.calls = []
|
|
|
|
def build_request(self, method, url, headers=None, json=None, content=None):
|
|
return FakeRequest(method, url, json)
|
|
|
|
async def send(self, request, stream=False):
|
|
self.calls.append((request.url, request.json_body))
|
|
return FakeResp_openai_gateway(status=500, payload={"error": "boom"})
|
|
|
|
async def aclose(self):
|
|
pass
|
|
|
|
|
|
class FakeBadRequestClient_openai_gateway:
|
|
def __init__(self, *a, **k):
|
|
self.calls = []
|
|
|
|
def build_request(self, method, url, headers=None, json=None, content=None):
|
|
return FakeRequest(method, url, json)
|
|
|
|
async def send(self, request, stream=False):
|
|
self.calls.append((request.url, request.json_body))
|
|
return FakeResp_openai_gateway(
|
|
status=400, payload={"error": {"message": "invalid request: bad param"}}
|
|
)
|
|
|
|
async def aclose(self):
|
|
pass
|
|
|
|
|
|
def test_chat_does_not_fall_back_on_an_unrecoverable_bad_request(local_db, monkeypatch):
|
|
from devplacepy.services.openai_gateway import routing
|
|
|
|
monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeBadRequestClient_openai_gateway)
|
|
routing.model_store.set(
|
|
routing.ModelRouteIn(source_model="fb-badreq-backup", target_model="backup-target")
|
|
)
|
|
routing.model_store.set(
|
|
routing.ModelRouteIn(
|
|
source_model="fb-badreq-primary",
|
|
target_model="primary-target",
|
|
fallback_model="fb-badreq-backup",
|
|
)
|
|
)
|
|
try:
|
|
svc = GatewayService()
|
|
cfg = svc.effective_config()
|
|
cfg["gateway_vision_enabled"] = False
|
|
cfg["gateway_max_retries"] = 0
|
|
rt = svc.runtime()
|
|
response = run_async(
|
|
rt.handle_chat(
|
|
{"model": "fb-badreq-primary", "messages": [{"role": "user", "content": "hi"}]},
|
|
cfg,
|
|
("guest", "no_fallback_on_bad_request"),
|
|
"test",
|
|
"default",
|
|
)
|
|
)
|
|
assert response.status_code == 400
|
|
assert len(rt._client.calls) == 1
|
|
finally:
|
|
routing.model_store.remove("fb-badreq-primary")
|
|
routing.model_store.remove("fb-badreq-backup")
|
|
|
|
|
|
def test_chat_falls_back_when_the_primary_model_fails(local_db, monkeypatch):
|
|
from devplacepy.services.openai_gateway import routing
|
|
|
|
monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeFallbackClient_openai_gateway)
|
|
routing.model_store.set(
|
|
routing.ModelRouteIn(source_model="fb-backup-route", target_model="backup-target")
|
|
)
|
|
routing.model_store.set(
|
|
routing.ModelRouteIn(
|
|
source_model="fb-primary-route",
|
|
target_model="primary-target",
|
|
fallback_model="fb-backup-route",
|
|
)
|
|
)
|
|
try:
|
|
svc = GatewayService()
|
|
cfg = svc.effective_config()
|
|
cfg["gateway_vision_enabled"] = False
|
|
cfg["gateway_max_retries"] = 0
|
|
rt = svc.runtime()
|
|
response = run_async(
|
|
rt.handle_chat(
|
|
{"model": "fb-primary-route", "messages": [{"role": "user", "content": "hi"}]},
|
|
cfg,
|
|
("guest", "fallback_chat_success"),
|
|
"test",
|
|
"default",
|
|
)
|
|
)
|
|
assert response.status_code == 200
|
|
assert rt._client.calls[0][1]["model"] == "primary-target"
|
|
assert rt._client.calls[-1][1]["model"] == "backup-target"
|
|
row = get_table("gateway_usage_ledger").find_one(owner_id="fallback_chat_success")
|
|
assert row is not None
|
|
assert row["requested_model"] == "fb-primary-route"
|
|
assert row["model"] == "backup-target"
|
|
assert row["success"] == 1
|
|
finally:
|
|
routing.model_store.remove("fb-primary-route")
|
|
routing.model_store.remove("fb-backup-route")
|
|
|
|
|
|
class FakeChainClient_openai_gateway:
|
|
def __init__(self, *a, **k):
|
|
self.calls = []
|
|
|
|
def build_request(self, method, url, headers=None, json=None, content=None):
|
|
return FakeRequest(method, url, json)
|
|
|
|
async def send(self, request, stream=False):
|
|
self.calls.append((request.url, request.json_body))
|
|
body = request.json_body or {}
|
|
model = body.get("model")
|
|
if model in ("chain-a-target", "chain-b-target"):
|
|
return FakeResp_openai_gateway(status=500, payload={"error": "boom"})
|
|
return FakeResp_openai_gateway(
|
|
payload={
|
|
"id": "x",
|
|
"model": model,
|
|
"choices": [{"message": {"content": "chain ok"}}],
|
|
}
|
|
)
|
|
|
|
async def aclose(self):
|
|
pass
|
|
|
|
|
|
def test_chat_walks_fallback_chain_past_failures(local_db, monkeypatch):
|
|
from devplacepy.services.openai_gateway import routing
|
|
|
|
monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeChainClient_openai_gateway)
|
|
routing.model_store.set(
|
|
routing.ModelRouteIn(source_model="chain-c", target_model="chain-c-target")
|
|
)
|
|
routing.model_store.set(
|
|
routing.ModelRouteIn(
|
|
source_model="chain-b",
|
|
target_model="chain-b-target",
|
|
fallback_model="chain-c",
|
|
)
|
|
)
|
|
routing.model_store.set(
|
|
routing.ModelRouteIn(
|
|
source_model="chain-a",
|
|
target_model="chain-a-target",
|
|
fallback_model="chain-b",
|
|
)
|
|
)
|
|
try:
|
|
svc = GatewayService()
|
|
cfg = svc.effective_config()
|
|
cfg["gateway_vision_enabled"] = False
|
|
cfg["gateway_max_retries"] = 0
|
|
rt = svc.runtime()
|
|
response = run_async(
|
|
rt.handle_chat(
|
|
{"model": "chain-a", "messages": [{"role": "user", "content": "hi"}]},
|
|
cfg,
|
|
("guest", "chain_walk"),
|
|
"test",
|
|
"default",
|
|
)
|
|
)
|
|
assert response.status_code == 200
|
|
assert [c[1]["model"] for c in rt._client.calls] == [
|
|
"chain-a-target",
|
|
"chain-b-target",
|
|
"chain-c-target",
|
|
]
|
|
row = get_table("gateway_usage_ledger").find_one(owner_id="chain_walk")
|
|
assert row["model"] == "chain-c-target"
|
|
assert row["fallback_used_route"] == "chain-c"
|
|
finally:
|
|
routing.model_store.remove("chain-a")
|
|
routing.model_store.remove("chain-b")
|
|
routing.model_store.remove("chain-c")
|
|
|
|
|
|
def test_chat_preroutes_around_unhealthy_models(local_db, monkeypatch):
|
|
from devplacepy.services.openai_gateway import health, routing
|
|
|
|
monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeClient_openai_gateway)
|
|
routing.model_store.set(
|
|
routing.ModelRouteIn(source_model="pre-good", target_model="good-target")
|
|
)
|
|
routing.model_store.set(
|
|
routing.ModelRouteIn(
|
|
source_model="pre-bad",
|
|
target_model="bad-target",
|
|
fallback_model="pre-good",
|
|
)
|
|
)
|
|
health.record_health(
|
|
"pre-bad", health.ProbeResult(False, "http_500", "boom", 1.0)
|
|
)
|
|
try:
|
|
svc = GatewayService()
|
|
cfg = svc.effective_config()
|
|
cfg["gateway_vision_enabled"] = False
|
|
rt = svc.runtime()
|
|
logged = []
|
|
response = run_async(
|
|
rt.handle_chat(
|
|
{"model": "pre-bad", "messages": [{"role": "user", "content": "hi"}]},
|
|
cfg,
|
|
("guest", "preroute"),
|
|
"test",
|
|
"default",
|
|
log=logged.append,
|
|
)
|
|
)
|
|
assert response.status_code == 200
|
|
assert len(rt._client.calls) == 1
|
|
assert rt._client.calls[0][1]["model"] == "good-target"
|
|
assert any("is unhealthy, starting on" in line for line in logged)
|
|
row = get_table("gateway_usage_ledger").find_one(owner_id="preroute")
|
|
assert row["requested_model"] == "pre-bad"
|
|
assert row["model"] == "good-target"
|
|
finally:
|
|
routing.model_store.remove("pre-bad")
|
|
routing.model_store.remove("pre-good")
|
|
|
|
|
|
def test_chat_tries_original_when_chain_is_exhausted(local_db, monkeypatch):
|
|
from devplacepy.services.openai_gateway import health, routing
|
|
|
|
monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeClient_openai_gateway)
|
|
routing.model_store.set(
|
|
routing.ModelRouteIn(
|
|
source_model="exh-fb",
|
|
target_model="exh-fb-target",
|
|
fallback_model="exh-primary",
|
|
)
|
|
)
|
|
routing.model_store.set(
|
|
routing.ModelRouteIn(
|
|
source_model="exh-primary",
|
|
target_model="exh-primary-target",
|
|
fallback_model="exh-fb",
|
|
)
|
|
)
|
|
health.record_health(
|
|
"exh-primary", health.ProbeResult(False, "http_500", "boom", 1.0)
|
|
)
|
|
health.record_health(
|
|
"exh-fb", health.ProbeResult(False, "http_500", "boom", 1.0)
|
|
)
|
|
try:
|
|
svc = GatewayService()
|
|
cfg = svc.effective_config()
|
|
cfg["gateway_vision_enabled"] = False
|
|
rt = svc.runtime()
|
|
response = run_async(
|
|
rt.handle_chat(
|
|
{"model": "exh-primary", "messages": [{"role": "user", "content": "hi"}]},
|
|
cfg,
|
|
("guest", "exhausted"),
|
|
"test",
|
|
"default",
|
|
)
|
|
)
|
|
assert response.status_code == 200
|
|
assert [c[1]["model"] for c in rt._client.calls] == ["exh-primary-target"]
|
|
finally:
|
|
routing.model_store.remove("exh-primary")
|
|
routing.model_store.remove("exh-fb")
|
|
|
|
|
|
def test_models_endpoint_hides_unhealthy_routes(local_db):
|
|
from devplacepy.services.openai_gateway import health, routing
|
|
|
|
routing.model_store.set(
|
|
routing.ModelRouteIn(source_model="hide-me", target_model="v/hide")
|
|
)
|
|
health.record_health(
|
|
"hide-me", health.ProbeResult(False, "http_500", "boom", 1.0)
|
|
)
|
|
try:
|
|
svc = GatewayService()
|
|
payload = json.loads(svc._models_response().body.decode())
|
|
assert "hide-me" not in {m["id"] for m in payload["data"]}
|
|
health.record_health(
|
|
"hide-me", health.ProbeResult(True, "ok", "", 1.0)
|
|
)
|
|
payload = json.loads(svc._models_response().body.decode())
|
|
assert "hide-me" in {m["id"] for m in payload["data"]}
|
|
finally:
|
|
routing.model_store.remove("hide-me")
|
|
|
|
|
|
class FakeContextLengthClient_openai_gateway:
|
|
def __init__(self, *a, **k):
|
|
self.calls = []
|
|
|
|
def build_request(self, method, url, headers=None, json=None, content=None):
|
|
return FakeRequest(method, url, json)
|
|
|
|
async def send(self, request, stream=False):
|
|
self.calls.append((request.url, request.json_body))
|
|
body = request.json_body or {}
|
|
model = body.get("model")
|
|
if model == "ctx-primary-target":
|
|
return FakeResp_openai_gateway(
|
|
status=400,
|
|
payload={"error": {"message": "maximum context length exceeded"}},
|
|
)
|
|
return FakeResp_openai_gateway(
|
|
payload={
|
|
"id": "x",
|
|
"model": model,
|
|
"choices": [{"message": {"content": "fallback ok"}}],
|
|
}
|
|
)
|
|
|
|
async def aclose(self):
|
|
pass
|
|
|
|
|
|
def test_chat_falls_back_on_a_context_length_error(local_db, monkeypatch):
|
|
from devplacepy.services.openai_gateway import routing
|
|
|
|
monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeContextLengthClient_openai_gateway)
|
|
routing.model_store.set(
|
|
routing.ModelRouteIn(source_model="ctx-backup-route", target_model="ctx-backup-target")
|
|
)
|
|
routing.model_store.set(
|
|
routing.ModelRouteIn(
|
|
source_model="ctx-primary-route",
|
|
target_model="ctx-primary-target",
|
|
fallback_model="ctx-backup-route",
|
|
)
|
|
)
|
|
try:
|
|
svc = GatewayService()
|
|
cfg = svc.effective_config()
|
|
cfg["gateway_vision_enabled"] = False
|
|
cfg["gateway_max_retries"] = 0
|
|
rt = svc.runtime()
|
|
response = run_async(
|
|
rt.handle_chat(
|
|
{"model": "ctx-primary-route", "messages": [{"role": "user", "content": "hi"}]},
|
|
cfg,
|
|
("guest", "context_length_fallback_success"),
|
|
"test",
|
|
"default",
|
|
)
|
|
)
|
|
assert response.status_code == 200
|
|
assert rt._client.calls[0][1]["model"] == "ctx-primary-target"
|
|
assert rt._client.calls[-1][1]["model"] == "ctx-backup-target"
|
|
row = get_table("gateway_usage_ledger").find_one(
|
|
owner_id="context_length_fallback_success"
|
|
)
|
|
assert row["success"] == 1
|
|
assert row["fallback_used_route"] == "ctx-backup-route"
|
|
finally:
|
|
routing.model_store.remove("ctx-primary-route")
|
|
routing.model_store.remove("ctx-backup-route")
|
|
|
|
|
|
def test_chat_skips_a_too_small_primary_and_goes_straight_to_fallback(local_db, monkeypatch):
|
|
from devplacepy.services.openai_gateway import routing
|
|
|
|
monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeContextLengthClient_openai_gateway)
|
|
routing.model_store.set(
|
|
routing.ModelRouteIn(
|
|
source_model="ctx-precheck-backup",
|
|
target_model="ctx-backup-target",
|
|
context_window=1_000_000,
|
|
)
|
|
)
|
|
routing.model_store.set(
|
|
routing.ModelRouteIn(
|
|
source_model="ctx-precheck-primary",
|
|
target_model="ctx-primary-target",
|
|
context_window=50,
|
|
fallback_model="ctx-precheck-backup",
|
|
)
|
|
)
|
|
try:
|
|
svc = GatewayService()
|
|
cfg = svc.effective_config()
|
|
cfg["gateway_vision_enabled"] = False
|
|
cfg["gateway_max_retries"] = 0
|
|
rt = svc.runtime()
|
|
long_message = "word " * 2000
|
|
response = run_async(
|
|
rt.handle_chat(
|
|
{
|
|
"model": "ctx-precheck-primary",
|
|
"messages": [{"role": "user", "content": long_message}],
|
|
},
|
|
cfg,
|
|
("guest", "context_precheck_skips_primary"),
|
|
"test",
|
|
"default",
|
|
)
|
|
)
|
|
assert response.status_code == 200
|
|
# Only the fallback was ever called - the primary was known too small.
|
|
assert len(rt._client.calls) == 1
|
|
assert rt._client.calls[0][1]["model"] == "ctx-backup-target"
|
|
row = get_table("gateway_usage_ledger").find_one(
|
|
owner_id="context_precheck_skips_primary"
|
|
)
|
|
assert row["fallback_used_route"] == "ctx-precheck-backup"
|
|
finally:
|
|
routing.model_store.remove("ctx-precheck-primary")
|
|
routing.model_store.remove("ctx-precheck-backup")
|
|
|
|
|
|
def test_chat_falls_back_via_molodetz_when_the_client_sends_an_unrouted_model_name(
|
|
local_db, monkeypatch
|
|
):
|
|
from devplacepy.services.openai_gateway import routing
|
|
|
|
original = routing.model_store.get("molodetz")
|
|
routing.model_store.set(
|
|
routing.ModelRouteIn(source_model="fb-molodetz-backup", target_model="backup-target")
|
|
)
|
|
routing.model_store.set(
|
|
routing.ModelRouteIn(
|
|
source_model="molodetz",
|
|
target_model="primary-target",
|
|
fallback_model="fb-molodetz-backup",
|
|
)
|
|
)
|
|
try:
|
|
svc = GatewayService()
|
|
cfg = svc.effective_config()
|
|
cfg["gateway_vision_enabled"] = False
|
|
cfg["gateway_max_retries"] = 0
|
|
cfg["gateway_force_model"] = False
|
|
base_default_model = cfg["gateway_model"]
|
|
|
|
class FakeUnroutedPrimaryFailsClient:
|
|
def __init__(self, *a, **k):
|
|
self.calls = []
|
|
|
|
def build_request(self, method, url, headers=None, json=None, content=None):
|
|
return FakeRequest(method, url, json)
|
|
|
|
async def send(self, request, stream=False):
|
|
self.calls.append((request.url, request.json_body))
|
|
body = request.json_body or {}
|
|
model = body.get("model")
|
|
if model == base_default_model:
|
|
return FakeResp_openai_gateway(
|
|
status=402, payload={"error": {"message": "Insufficient Balance"}}
|
|
)
|
|
return FakeResp_openai_gateway(
|
|
payload={
|
|
"id": "x",
|
|
"model": model,
|
|
"choices": [{"message": {"content": "fallback ok"}}],
|
|
}
|
|
)
|
|
|
|
async def aclose(self):
|
|
pass
|
|
|
|
monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeUnroutedPrimaryFailsClient)
|
|
rt = svc.runtime()
|
|
# The client sends a model name that matches no configured route at all
|
|
# (as a caller like dpc does, which is never told the exact "molodetz"
|
|
# routing alias) - the primary call falls through to the flat/global
|
|
# default model (no route overlay applies), but a failure there must
|
|
# still consult molodetz's own configured fallback, not silently skip
|
|
# it just because the client's literal string never matched anything.
|
|
response = run_async(
|
|
rt.handle_chat(
|
|
{
|
|
"model": "some-client-side-model-name",
|
|
"messages": [{"role": "user", "content": "hi"}],
|
|
},
|
|
cfg,
|
|
("guest", "fallback_chat_unrouted_client_model"),
|
|
"test",
|
|
"default",
|
|
)
|
|
)
|
|
assert response.status_code == 200
|
|
assert rt._client.calls[0][1]["model"] == base_default_model
|
|
assert rt._client.calls[-1][1]["model"] == "backup-target"
|
|
row = get_table("gateway_usage_ledger").find_one(
|
|
owner_id="fallback_chat_unrouted_client_model"
|
|
)
|
|
assert row is not None
|
|
assert row["requested_model"] == "some-client-side-model-name"
|
|
assert row["model"] == "backup-target"
|
|
assert row["success"] == 1
|
|
finally:
|
|
routing.model_store.remove("fb-molodetz-backup")
|
|
if original is not None:
|
|
routing.model_store.set(
|
|
routing.ModelRouteIn(
|
|
**{
|
|
field: getattr(original, field)
|
|
for field in routing.ModelRouteIn.model_fields
|
|
}
|
|
)
|
|
)
|
|
else:
|
|
routing.model_store.remove("molodetz")
|
|
|
|
|
|
def test_chat_returns_the_fallback_failure_when_both_models_fail(local_db, monkeypatch):
|
|
from devplacepy.services.openai_gateway import routing
|
|
|
|
monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeAlwaysFailClient_openai_gateway)
|
|
routing.model_store.set(
|
|
routing.ModelRouteIn(source_model="fb-backup-route2", target_model="backup-target2")
|
|
)
|
|
routing.model_store.set(
|
|
routing.ModelRouteIn(
|
|
source_model="fb-primary-route2",
|
|
target_model="primary-target2",
|
|
fallback_model="fb-backup-route2",
|
|
)
|
|
)
|
|
try:
|
|
svc = GatewayService()
|
|
cfg = svc.effective_config()
|
|
cfg["gateway_vision_enabled"] = False
|
|
cfg["gateway_max_retries"] = 0
|
|
rt = svc.runtime()
|
|
response = run_async(
|
|
rt.handle_chat(
|
|
{"model": "fb-primary-route2", "messages": [{"role": "user", "content": "hi"}]},
|
|
cfg,
|
|
("guest", "fallback_chat_both_fail"),
|
|
"test",
|
|
"default",
|
|
)
|
|
)
|
|
assert response.status_code == 500
|
|
assert len(rt._client.calls) == 2
|
|
assert rt._client.calls[0][1]["model"] == "primary-target2"
|
|
assert rt._client.calls[1][1]["model"] == "backup-target2"
|
|
finally:
|
|
routing.model_store.remove("fb-primary-route2")
|
|
routing.model_store.remove("fb-backup-route2")
|
|
|
|
|
|
def test_chat_without_a_fallback_never_makes_a_second_call(local_db, monkeypatch):
|
|
from devplacepy.services.openai_gateway import routing
|
|
|
|
monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeAlwaysFailClient_openai_gateway)
|
|
routing.model_store.set(
|
|
routing.ModelRouteIn(source_model="fb-no-fallback", target_model="no-fallback-target")
|
|
)
|
|
try:
|
|
svc = GatewayService()
|
|
cfg = svc.effective_config()
|
|
cfg["gateway_vision_enabled"] = False
|
|
cfg["gateway_max_retries"] = 0
|
|
rt = svc.runtime()
|
|
run_async(
|
|
rt.handle_chat(
|
|
{"model": "fb-no-fallback", "messages": [{"role": "user", "content": "hi"}]},
|
|
cfg,
|
|
("guest", "no_fallback_configured"),
|
|
"test",
|
|
"default",
|
|
)
|
|
)
|
|
assert len(rt._client.calls) == 1
|
|
finally:
|
|
routing.model_store.remove("fb-no-fallback")
|
|
|
|
|
|
def test_embeddings_unknown_model_falls_back_to_default(local_db, monkeypatch):
|
|
monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeEmbedClient_openai_gateway)
|
|
monkeypatch.setenv("OPENROUTER_API_KEY", "test-embed-key")
|
|
svc = GatewayService()
|
|
cfg = svc.effective_config()
|
|
cfg["gateway_force_model"] = False
|
|
cfg["gateway_embed_enabled"] = True
|
|
cfg["gateway_embed_model"] = "qwen/qwen3-embedding-8b"
|
|
rt = svc.runtime()
|
|
run_async(
|
|
rt.handle_embeddings(
|
|
{"model": "nonexistent-embed-model", "input": "hello"},
|
|
cfg,
|
|
("guest", "fallback_test_embed"),
|
|
"test",
|
|
"default",
|
|
)
|
|
)
|
|
assert rt._client.calls[-1][1]["model"] == "qwen/qwen3-embedding-8b"
|
|
|
|
|
|
def test_images_unknown_model_falls_back_to_default(local_db, monkeypatch):
|
|
monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeImageClient_openai_gateway)
|
|
svc = GatewayService()
|
|
cfg = svc.effective_config()
|
|
cfg["gateway_force_model"] = False
|
|
cfg["gateway_image_enabled"] = True
|
|
cfg["gateway_image_model"] = "black-forest-labs/flux.2-pro"
|
|
rt = svc.runtime()
|
|
run_async(
|
|
rt.handle_images(
|
|
{"model": "nonexistent-image-model", "prompt": "test badge"},
|
|
cfg,
|
|
("guest", "fallback_test_img"),
|
|
"test",
|
|
"default",
|
|
)
|
|
)
|
|
assert rt._client.calls[-1][1]["model"] == "black-forest-labs/flux.2-pro"
|
|
|
|
|
|
class FakeClientWithUsage_openai_gateway(FakeClient_openai_gateway):
|
|
async def send(self, request, stream=False):
|
|
self.calls.append((request.url, request.json_body))
|
|
body = request.json_body or {}
|
|
return FakeResp_openai_gateway(
|
|
payload={
|
|
"id": "x",
|
|
"model": body.get("model"),
|
|
"choices": [{"message": {"content": "hi there"}}],
|
|
"usage": {"prompt_tokens": 7, "completion_tokens": 3, "total_tokens": 10},
|
|
}
|
|
)
|
|
|
|
|
|
def test_ollama_stream_does_not_send_stream_options(local_db, monkeypatch):
|
|
monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeClient_openai_gateway)
|
|
svc = GatewayService()
|
|
cfg = svc.effective_config()
|
|
cfg["gateway_upstream_url"] = "http://127.0.0.1:11434/api/chat"
|
|
rt = svc.runtime()
|
|
run_async(
|
|
rt.handle_chat(
|
|
{
|
|
"messages": [{"role": "user", "content": "hi"}],
|
|
"stream": True,
|
|
"stream_options": {"include_usage": True},
|
|
},
|
|
cfg,
|
|
("guest", "ollama_stream_opts"),
|
|
"test",
|
|
"default",
|
|
)
|
|
)
|
|
upstream_payload = rt._client.calls[-1][1]
|
|
assert upstream_payload["stream"] is True
|
|
assert "stream_options" not in upstream_payload
|
|
assert upstream_payload["think"] is False
|
|
|
|
|
|
def test_ollama_explicit_dialect_overrides_url(local_db, monkeypatch):
|
|
monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeClient_openai_gateway)
|
|
svc = GatewayService()
|
|
cfg = svc.effective_config()
|
|
cfg["gateway_upstream_url"] = "https://ai.example.com/v1/chat/completions"
|
|
cfg["gateway_thinking_dialect"] = "ollama"
|
|
rt = svc.runtime()
|
|
run_async(
|
|
rt.handle_chat(
|
|
{"messages": [{"role": "user", "content": "hi"}]},
|
|
cfg,
|
|
("guest", "ollama_dialect_override"),
|
|
"test",
|
|
"default",
|
|
)
|
|
)
|
|
body = rt._client.calls[-1][1]
|
|
assert body["think"] is False
|
|
assert "thinking" not in body
|
|
|
|
|
|
def test_client_model_allowed_when_allow_client_model_on(local_db, monkeypatch):
|
|
monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeClient_openai_gateway)
|
|
svc = GatewayService()
|
|
cfg = svc.effective_config()
|
|
cfg["gateway_force_model"] = True
|
|
cfg["gateway_allow_client_model"] = True
|
|
cfg["gateway_model"] = "deepseek-chat"
|
|
rt = svc.runtime()
|
|
run_async(
|
|
rt.handle_chat(
|
|
{"model": "llama3.2", "messages": [{"role": "user", "content": "hi"}]},
|
|
cfg,
|
|
("guest", "client_model"),
|
|
"test",
|
|
"default",
|
|
)
|
|
)
|
|
assert rt._client.calls[-1][1]["model"] == "llama3.2"
|
|
|
|
|
|
def test_molodetz_alias_still_remapped_with_allow_client_model(local_db, monkeypatch):
|
|
from devplacepy.services.openai_gateway import routing
|
|
|
|
monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeClient_openai_gateway)
|
|
original = routing.model_store.get("molodetz")
|
|
routing.model_store.remove("molodetz")
|
|
try:
|
|
svc = GatewayService()
|
|
cfg = svc.effective_config()
|
|
cfg["gateway_force_model"] = True
|
|
cfg["gateway_allow_client_model"] = True
|
|
cfg["gateway_model"] = "deepseek-chat"
|
|
rt = svc.runtime()
|
|
run_async(
|
|
rt.handle_chat(
|
|
{"model": "molodetz", "messages": [{"role": "user", "content": "hi"}]},
|
|
cfg,
|
|
("guest", "molodetz_alias"),
|
|
"test",
|
|
"default",
|
|
)
|
|
)
|
|
assert rt._client.calls[-1][1]["model"] == "deepseek-chat"
|
|
finally:
|
|
if original is not None:
|
|
routing.model_store.set(
|
|
routing.ModelRouteIn(
|
|
**{
|
|
field: getattr(original, field)
|
|
for field in routing.ModelRouteIn.model_fields
|
|
}
|
|
)
|
|
)
|
|
|
|
|
|
def test_stream_forwarded_to_upstream_with_forced_include_usage(local_db, monkeypatch):
|
|
monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeClient_openai_gateway)
|
|
svc = GatewayService()
|
|
cfg = svc.effective_config()
|
|
rt = svc.runtime()
|
|
run_async(
|
|
rt.handle_chat(
|
|
{
|
|
"messages": [{"role": "user", "content": "hi"}],
|
|
"stream": True,
|
|
"stream_options": {"include_usage": False},
|
|
},
|
|
cfg,
|
|
("guest", "test"),
|
|
"test",
|
|
"default",
|
|
)
|
|
)
|
|
upstream_payload = rt._client.calls[-1][1]
|
|
assert upstream_payload["stream"] is True
|
|
assert upstream_payload["stream_options"] == {"include_usage": True}
|
|
assert upstream_payload["thinking"] == {"type": "disabled"}
|
|
assert "think" not in upstream_payload
|
|
assert "reasoning" not in upstream_payload
|
|
|
|
|
|
def test_non_stream_has_no_stream_options_upstream(local_db, monkeypatch):
|
|
monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeClient_openai_gateway)
|
|
svc = GatewayService()
|
|
cfg = svc.effective_config()
|
|
rt = svc.runtime()
|
|
run_async(
|
|
rt.handle_chat(
|
|
{
|
|
"messages": [{"role": "user", "content": "hi"}],
|
|
"stream_options": {"include_usage": True},
|
|
},
|
|
cfg,
|
|
("guest", "test"),
|
|
"test",
|
|
"default",
|
|
)
|
|
)
|
|
upstream_payload = rt._client.calls[-1][1]
|
|
assert "stream_options" not in upstream_payload
|
|
assert upstream_payload["stream"] is False
|
|
|
|
|
|
def test_thinking_disabled_by_default(local_db, monkeypatch):
|
|
monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeClient_openai_gateway)
|
|
svc = GatewayService()
|
|
cfg = svc.effective_config()
|
|
rt = svc.runtime()
|
|
run_async(
|
|
rt.handle_chat(
|
|
{"messages": [{"role": "user", "content": "hi"}]},
|
|
cfg,
|
|
("guest", "test"),
|
|
"test",
|
|
"default",
|
|
)
|
|
)
|
|
body = rt._client.calls[-1][1]
|
|
assert body["thinking"] == {"type": "disabled"}
|
|
|
|
|
|
def test_client_think_true_enables_upstream_thinking(local_db, monkeypatch):
|
|
monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeClient_openai_gateway)
|
|
svc = GatewayService()
|
|
cfg = svc.effective_config()
|
|
rt = svc.runtime()
|
|
run_async(
|
|
rt.handle_chat(
|
|
{
|
|
"messages": [{"role": "user", "content": "hi"}],
|
|
"think": True,
|
|
},
|
|
cfg,
|
|
("guest", "test"),
|
|
"test",
|
|
"default",
|
|
)
|
|
)
|
|
body = rt._client.calls[-1][1]
|
|
assert body["thinking"] == {"type": "enabled"}
|
|
assert "think" not in body
|
|
|
|
|
|
def test_openrouter_upstream_gets_reasoning_none(local_db, monkeypatch):
|
|
monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeClient_openai_gateway)
|
|
svc = GatewayService()
|
|
cfg = svc.effective_config()
|
|
cfg["gateway_upstream_url"] = "https://openrouter.ai/api/v1/chat/completions"
|
|
rt = svc.runtime()
|
|
run_async(
|
|
rt.handle_chat(
|
|
{"messages": [{"role": "user", "content": "hi"}]},
|
|
cfg,
|
|
("guest", "test"),
|
|
"test",
|
|
"default",
|
|
)
|
|
)
|
|
body = rt._client.calls[-1][1]
|
|
assert body["reasoning"] == {"effort": "none"}
|
|
assert "thinking" not in body
|
|
|
|
|
|
def test_include_usage_emits_usage_chunk(local_db, monkeypatch):
|
|
monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeClientWithUsage_openai_gateway)
|
|
svc = GatewayService()
|
|
cfg = svc.effective_config()
|
|
rt = svc.runtime()
|
|
resp = run_async(
|
|
rt.handle_chat(
|
|
{
|
|
"messages": [{"role": "user", "content": "hi"}],
|
|
"stream": True,
|
|
"stream_options": {"include_usage": True},
|
|
},
|
|
cfg,
|
|
("guest", "test"),
|
|
"test",
|
|
"default",
|
|
)
|
|
)
|
|
|
|
async def drain():
|
|
out = []
|
|
async for chunk in resp.body_iterator:
|
|
out.append(chunk if isinstance(chunk, str) else chunk.decode())
|
|
return "".join(out)
|
|
|
|
body = run_async(drain())
|
|
assert '"usage"' in body
|
|
assert '"total_tokens": 10' in body
|
|
assert "[DONE]" in body
|
|
|
|
|
|
def test_no_usage_chunk_without_include_usage(local_db, monkeypatch):
|
|
monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeClientWithUsage_openai_gateway)
|
|
svc = GatewayService()
|
|
cfg = svc.effective_config()
|
|
rt = svc.runtime()
|
|
resp = run_async(
|
|
rt.handle_chat(
|
|
{"messages": [{"role": "user", "content": "hi"}], "stream": True},
|
|
cfg,
|
|
("guest", "test"),
|
|
"test",
|
|
"default",
|
|
)
|
|
)
|
|
|
|
async def drain():
|
|
out = []
|
|
async for chunk in resp.body_iterator:
|
|
out.append(chunk if isinstance(chunk, str) else chunk.decode())
|
|
return "".join(out)
|
|
|
|
body = run_async(drain())
|
|
assert '"usage"' not in body
|
|
assert "[DONE]" in body
|
|
|
|
|
|
def test_stream_headers_carry_no_cost_or_token_data(local_db, monkeypatch):
|
|
monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeClientWithUsage_openai_gateway)
|
|
svc = GatewayService()
|
|
cfg = svc.effective_config()
|
|
rt = svc.runtime()
|
|
resp = run_async(
|
|
rt.handle_chat(
|
|
{"messages": [{"role": "user", "content": "hi"}], "stream": True},
|
|
cfg,
|
|
("guest", "test"),
|
|
"test",
|
|
"my-app",
|
|
)
|
|
)
|
|
assert resp.headers["X-Gateway-Model"] == cfg["gateway_model"]
|
|
assert resp.headers["X-Gateway-Backend"] == "chat"
|
|
assert resp.headers["X-App-Reference"] == "my-app"
|
|
assert "x-gateway-cost-usd" not in {k.lower() for k in resp.headers}
|
|
assert "x-gateway-total-tokens" not in {k.lower() for k in resp.headers}
|
|
|
|
|
|
def test_stream_records_ttft_and_inter_token_in_ledger(local_db, monkeypatch):
|
|
monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeClientWithUsage_openai_gateway)
|
|
svc = GatewayService()
|
|
cfg = svc.effective_config()
|
|
rt = svc.runtime()
|
|
resp = run_async(
|
|
rt.handle_chat(
|
|
{
|
|
"messages": [{"role": "user", "content": "hi"}],
|
|
"stream": True,
|
|
"stream_options": {"include_usage": True},
|
|
},
|
|
cfg,
|
|
("guest", "ttft_probe"),
|
|
"test",
|
|
"default",
|
|
)
|
|
)
|
|
|
|
async def drain():
|
|
async for _ in resp.body_iterator:
|
|
pass
|
|
|
|
run_async(drain())
|
|
|
|
row = get_table("gateway_usage_ledger").find_one(owner_id="ttft_probe")
|
|
assert row is not None
|
|
assert row["backend"] == "chat"
|
|
assert row["stream_requested"] == 1
|
|
assert row["success"] == 1
|
|
assert row["total_tokens"] == 10
|
|
assert row["ttft_ms"] is not None and row["ttft_ms"] >= 0.0
|
|
assert row["inter_token_ms"] is not None and row["inter_token_ms"] >= 0.0
|
|
|
|
|
|
class FakeClientReasoning_openai_gateway(FakeClient_openai_gateway):
|
|
async def send(self, request, stream=False):
|
|
self.calls.append((request.url, request.json_body))
|
|
body = request.json_body or {}
|
|
return FakeResp_openai_gateway(
|
|
payload={
|
|
"id": "x",
|
|
"model": body.get("model"),
|
|
"choices": [{"message": {"content": "x", "reasoning": "aaaaa"}}],
|
|
"usage": {"prompt_tokens": 7, "completion_tokens": 3, "total_tokens": 10},
|
|
}
|
|
)
|
|
|
|
|
|
def test_ollama_reasoning_delta_is_counted_as_a_content_chunk(local_db, monkeypatch):
|
|
monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeClientReasoning_openai_gateway)
|
|
svc = GatewayService()
|
|
cfg = svc.effective_config()
|
|
rt = svc.runtime()
|
|
resp = run_async(
|
|
rt.handle_chat(
|
|
{
|
|
"messages": [{"role": "user", "content": "hi"}],
|
|
"stream": True,
|
|
"stream_options": {"include_usage": True},
|
|
},
|
|
cfg,
|
|
("guest", "ollama_reasoning_probe"),
|
|
"test",
|
|
"default",
|
|
)
|
|
)
|
|
|
|
async def drain():
|
|
async for _ in resp.body_iterator:
|
|
pass
|
|
|
|
run_async(drain())
|
|
|
|
row = get_table("gateway_usage_ledger").find_one(owner_id="ollama_reasoning_probe")
|
|
assert row is not None
|
|
assert row["inter_token_ms"] is not None and row["inter_token_ms"] >= 0.0
|
|
|
|
|
|
def test_stream_client_disconnect_records_failure_and_closes_upstream(local_db, monkeypatch):
|
|
monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeClient_openai_gateway)
|
|
svc = GatewayService()
|
|
cfg = svc.effective_config()
|
|
rt = svc.runtime()
|
|
resp = run_async(
|
|
rt.handle_chat(
|
|
{"messages": [{"role": "user", "content": "hi"}], "stream": True},
|
|
cfg,
|
|
("guest", "disconnect_probe"),
|
|
"test",
|
|
"default",
|
|
)
|
|
)
|
|
|
|
async def partial_then_close():
|
|
agen = resp.body_iterator
|
|
await agen.__anext__()
|
|
await agen.aclose()
|
|
|
|
run_async(partial_then_close())
|
|
|
|
row = get_table("gateway_usage_ledger").find_one(owner_id="disconnect_probe")
|
|
assert row is not None
|
|
assert row["success"] == 0
|
|
assert row["error_category"] == "client_disconnected"
|
|
|
|
|
|
def test_models_endpoint_publishes_molodetz(local_db):
|
|
from devplacepy.services.openai_gateway import routing
|
|
|
|
routing.seed_default_deepseek_routes()
|
|
try:
|
|
svc = GatewayService()
|
|
resp = svc._models_response()
|
|
payload = json.loads(resp.body.decode())
|
|
ids = {m["id"] for m in payload["data"]}
|
|
assert payload["object"] == "list"
|
|
assert {"molodetz", "molodetz-pro"} <= ids
|
|
for model in payload["data"]:
|
|
assert model["object"] == "model"
|
|
assert model["owned_by"] == "molodetz"
|
|
finally:
|
|
routing.model_store.remove("molodetz")
|
|
routing.model_store.remove("molodetz-pro")
|
|
|
|
|
|
# Upstream header collision policy: only X-Gateway-Model is ever trusted and
|
|
# forwarded from an upstream that happens to speak our own X-Gateway-* header
|
|
# convention (e.g. another DevPlace-style gateway). Every other figure
|
|
# (cost/tokens/latency/context/app-reference) must remain ours regardless of
|
|
# what an upstream claims. See usage.upstream_reported_model / CLAUDE.md.
|
|
|
|
|
|
class FakeClientUpstreamModelHeader_openai_gateway(FakeClient_openai_gateway):
|
|
async def send(self, request, stream=False):
|
|
self.calls.append((request.url, request.json_body))
|
|
body = request.json_body or {}
|
|
return FakeResp_openai_gateway(
|
|
payload={
|
|
"id": "x",
|
|
"model": body.get("model"),
|
|
"choices": [{"message": {"content": "hi there"}}],
|
|
},
|
|
extra_headers={"X-Gateway-Model": "upstream/served-model-x"},
|
|
)
|
|
|
|
|
|
class FakeClientHostileModelHeader_openai_gateway(FakeClient_openai_gateway):
|
|
async def send(self, request, stream=False):
|
|
self.calls.append((request.url, request.json_body))
|
|
body = request.json_body or {}
|
|
return FakeResp_openai_gateway(
|
|
payload={
|
|
"id": "x",
|
|
"model": body.get("model"),
|
|
"choices": [{"message": {"content": "hi there"}}],
|
|
},
|
|
extra_headers={"X-Gateway-Model": "evil\r\nX-Injected: true"},
|
|
)
|
|
|
|
|
|
class FakeEmbedClientUpstreamModelHeader_openai_gateway(FakeEmbedClient_openai_gateway):
|
|
async def send(self, request, stream=False):
|
|
self.calls.append((request.url, request.json_body))
|
|
body = request.json_body or {}
|
|
return FakeResp_openai_gateway(
|
|
payload={
|
|
"object": "list",
|
|
"model": body.get("model"),
|
|
"data": [{"object": "embedding", "index": 0, "embedding": [0.1, 0.2]}],
|
|
"usage": {"prompt_tokens": 5, "total_tokens": 5},
|
|
},
|
|
extra_headers={"X-Gateway-Model": "upstream/embed-served"},
|
|
)
|
|
|
|
|
|
class FakeImageClientUpstreamModelHeader_openai_gateway(FakeImageClient_openai_gateway):
|
|
async def send(self, request, stream=False):
|
|
self.calls.append((request.url, request.json_body))
|
|
body = request.json_body or {}
|
|
return FakeResp_openai_gateway(
|
|
payload={
|
|
"created": 1,
|
|
"model": body.get("model"),
|
|
"data": [{"b64_json": "aGVsbG8="}],
|
|
"usage": {"cost": 0.05},
|
|
},
|
|
extra_headers={"X-Gateway-Model": "upstream/image-served"},
|
|
)
|
|
|
|
|
|
class FakePassthroughClientUpstreamModelHeader_openai_gateway:
|
|
def __init__(self, *a, **k):
|
|
self.calls = []
|
|
|
|
def build_request(self, method, url, headers=None, json=None, content=None):
|
|
return FakeRequest(method, url, json)
|
|
|
|
async def send(self, request, stream=False):
|
|
self.calls.append((request.url, request.json_body))
|
|
return FakeResp_openai_gateway(
|
|
payload={"ok": True},
|
|
extra_headers={"X-Gateway-Model": "upstream/passthrough-served"},
|
|
)
|
|
|
|
async def aclose(self):
|
|
pass
|
|
|
|
|
|
def test_upstream_x_gateway_model_header_is_forwarded_for_chat(local_db, monkeypatch):
|
|
monkeypatch.setattr(
|
|
gwmod.httpx, "AsyncClient", FakeClientUpstreamModelHeader_openai_gateway
|
|
)
|
|
svc = GatewayService()
|
|
cfg = svc.effective_config()
|
|
cfg["gateway_force_model"] = True
|
|
cfg["gateway_model"] = "deepseek-chat"
|
|
rt = svc.runtime()
|
|
resp = run_async(
|
|
rt.handle_chat(
|
|
{"messages": [{"role": "user", "content": "hi"}]},
|
|
cfg,
|
|
("guest", "upstream_model_chat"),
|
|
"test",
|
|
"default",
|
|
)
|
|
)
|
|
assert resp.headers["X-Gateway-Model"] == "upstream/served-model-x"
|
|
row = get_table("gateway_usage_ledger").find_one(owner_id="upstream_model_chat")
|
|
assert row is not None
|
|
assert row["model"] == "deepseek-chat"
|
|
|
|
|
|
def test_upstream_x_gateway_model_header_is_forwarded_for_streaming_chat(
|
|
local_db, monkeypatch
|
|
):
|
|
monkeypatch.setattr(
|
|
gwmod.httpx, "AsyncClient", FakeClientUpstreamModelHeader_openai_gateway
|
|
)
|
|
svc = GatewayService()
|
|
cfg = svc.effective_config()
|
|
rt = svc.runtime()
|
|
resp = run_async(
|
|
rt.handle_chat(
|
|
{"messages": [{"role": "user", "content": "hi"}], "stream": True},
|
|
cfg,
|
|
("guest", "upstream_model_stream"),
|
|
"test",
|
|
"default",
|
|
)
|
|
)
|
|
assert resp.headers["X-Gateway-Model"] == "upstream/served-model-x"
|
|
assert resp.headers["X-Gateway-Backend"] == "chat"
|
|
|
|
async def drain():
|
|
async for _ in resp.body_iterator:
|
|
pass
|
|
|
|
run_async(drain())
|
|
|
|
|
|
def test_upstream_x_gateway_model_header_is_forwarded_for_embeddings(
|
|
local_db, monkeypatch
|
|
):
|
|
monkeypatch.setattr(
|
|
gwmod.httpx, "AsyncClient", FakeEmbedClientUpstreamModelHeader_openai_gateway
|
|
)
|
|
monkeypatch.setenv("OPENROUTER_API_KEY", "test-embed-key")
|
|
svc = GatewayService()
|
|
cfg = svc.effective_config()
|
|
cfg["gateway_embed_enabled"] = True
|
|
cfg["gateway_embed_model"] = "qwen/qwen3-embedding-8b"
|
|
rt = svc.runtime()
|
|
resp = run_async(
|
|
rt.handle_embeddings(
|
|
{"input": "hello"},
|
|
cfg,
|
|
("guest", "upstream_model_embed"),
|
|
"test",
|
|
"default",
|
|
)
|
|
)
|
|
assert resp.headers["X-Gateway-Model"] == "upstream/embed-served"
|
|
row = get_table("gateway_usage_ledger").find_one(owner_id="upstream_model_embed")
|
|
assert row is not None
|
|
assert row["model"] == "qwen/qwen3-embedding-8b"
|
|
|
|
|
|
def test_upstream_x_gateway_model_header_is_forwarded_for_images(local_db, monkeypatch):
|
|
monkeypatch.setattr(
|
|
gwmod.httpx, "AsyncClient", FakeImageClientUpstreamModelHeader_openai_gateway
|
|
)
|
|
svc = GatewayService()
|
|
cfg = svc.effective_config()
|
|
cfg["gateway_image_enabled"] = True
|
|
cfg["gateway_image_model"] = "black-forest-labs/flux.2-pro"
|
|
rt = svc.runtime()
|
|
resp = run_async(
|
|
rt.handle_images(
|
|
{"prompt": "badge"},
|
|
cfg,
|
|
("guest", "upstream_model_img"),
|
|
"test",
|
|
"default",
|
|
)
|
|
)
|
|
assert resp.headers["X-Gateway-Model"] == "upstream/image-served"
|
|
row = get_table("gateway_usage_ledger").find_one(owner_id="upstream_model_img")
|
|
assert row is not None
|
|
assert row["model"] == "black-forest-labs/flux.2-pro"
|
|
|
|
|
|
def test_upstream_x_gateway_model_header_is_forwarded_for_passthrough(
|
|
local_db, monkeypatch
|
|
):
|
|
monkeypatch.setattr(
|
|
gwmod.httpx, "AsyncClient", FakePassthroughClientUpstreamModelHeader_openai_gateway
|
|
)
|
|
svc = GatewayService()
|
|
cfg = svc.effective_config()
|
|
cfg["gateway_model"] = "deepseek-chat"
|
|
rt = svc.runtime()
|
|
resp = run_async(
|
|
rt.handle_passthrough(
|
|
"POST",
|
|
"responses",
|
|
"application/json",
|
|
b"{}",
|
|
cfg,
|
|
("guest", "upstream_model_passthrough"),
|
|
"test",
|
|
"default",
|
|
)
|
|
)
|
|
assert resp.headers["X-Gateway-Model"] == "upstream/passthrough-served"
|
|
row = get_table("gateway_usage_ledger").find_one(
|
|
owner_id="upstream_model_passthrough"
|
|
)
|
|
assert row is not None
|
|
assert row["model"] == "deepseek-chat"
|
|
|
|
|
|
def test_no_upstream_model_header_keeps_our_own_model(local_db, monkeypatch):
|
|
monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeClient_openai_gateway)
|
|
svc = GatewayService()
|
|
cfg = svc.effective_config()
|
|
cfg["gateway_force_model"] = True
|
|
cfg["gateway_model"] = "deepseek-chat"
|
|
rt = svc.runtime()
|
|
resp = run_async(
|
|
rt.handle_chat(
|
|
{"messages": [{"role": "user", "content": "hi"}]},
|
|
cfg,
|
|
("guest", "no_upstream_header"),
|
|
"test",
|
|
"default",
|
|
)
|
|
)
|
|
assert resp.headers["X-Gateway-Model"] == "deepseek-chat"
|
|
|
|
|
|
class FakeEmbeddedErrorClient_openai_gateway:
|
|
"""Mimics OpenRouter's documented behavior of answering 200 OK with the
|
|
failure embedded in the JSON body (openrouter.ai/docs/api-reference/errors)
|
|
instead of a non-2xx status."""
|
|
|
|
def __init__(self, *a, **k):
|
|
self.calls = []
|
|
|
|
def build_request(self, method, url, headers=None, json=None, content=None):
|
|
return FakeRequest(method, url, json)
|
|
|
|
async def send(self, request, stream=False):
|
|
self.calls.append((request.url, request.json_body))
|
|
body = request.json_body or {}
|
|
model = body.get("model")
|
|
if model == "primary-target":
|
|
return FakeResp_openai_gateway(
|
|
status=200,
|
|
payload={"error": {"message": "no provider available", "code": 502}},
|
|
)
|
|
return FakeResp_openai_gateway(
|
|
payload={
|
|
"id": "x",
|
|
"model": model,
|
|
"choices": [{"message": {"content": "fallback ok"}}],
|
|
}
|
|
)
|
|
|
|
async def aclose(self):
|
|
pass
|
|
|
|
|
|
def test_chat_falls_back_when_upstream_returns_200_with_an_embedded_error(
|
|
local_db, monkeypatch
|
|
):
|
|
from devplacepy.services.openai_gateway import routing
|
|
|
|
monkeypatch.setattr(
|
|
gwmod.httpx, "AsyncClient", FakeEmbeddedErrorClient_openai_gateway
|
|
)
|
|
routing.model_store.set(
|
|
routing.ModelRouteIn(source_model="eb-backup-route", target_model="backup-target")
|
|
)
|
|
routing.model_store.set(
|
|
routing.ModelRouteIn(
|
|
source_model="eb-primary-route",
|
|
target_model="primary-target",
|
|
fallback_model="eb-backup-route",
|
|
)
|
|
)
|
|
try:
|
|
svc = GatewayService()
|
|
cfg = svc.effective_config()
|
|
cfg["gateway_vision_enabled"] = False
|
|
cfg["gateway_max_retries"] = 0
|
|
rt = svc.runtime()
|
|
response = run_async(
|
|
rt.handle_chat(
|
|
{"model": "eb-primary-route", "messages": [{"role": "user", "content": "hi"}]},
|
|
cfg,
|
|
("guest", "embedded_error_chat"),
|
|
"test",
|
|
"default",
|
|
)
|
|
)
|
|
assert response.status_code == 200
|
|
assert rt._client.calls[0][1]["model"] == "primary-target"
|
|
assert rt._client.calls[-1][1]["model"] == "backup-target"
|
|
row = get_table("gateway_usage_ledger").find_one(owner_id="embedded_error_chat")
|
|
assert row is not None
|
|
assert row["requested_model"] == "eb-primary-route"
|
|
assert row["model"] == "backup-target"
|
|
assert row["success"] == 1
|
|
finally:
|
|
routing.model_store.remove("eb-primary-route")
|
|
routing.model_store.remove("eb-backup-route")
|
|
|
|
|
|
class FakeAlwaysEmbeddedErrorClient_openai_gateway:
|
|
def __init__(self, *a, **k):
|
|
self.calls = []
|
|
|
|
def build_request(self, method, url, headers=None, json=None, content=None):
|
|
return FakeRequest(method, url, json)
|
|
|
|
async def send(self, request, stream=False):
|
|
self.calls.append((request.url, request.json_body))
|
|
return FakeResp_openai_gateway(
|
|
status=200,
|
|
payload={"error": {"message": "no provider available", "code": 502}},
|
|
)
|
|
|
|
async def aclose(self):
|
|
pass
|
|
|
|
|
|
def test_chat_reports_502_when_upstream_returns_200_with_an_embedded_error_and_no_fallback(
|
|
local_db, monkeypatch
|
|
):
|
|
monkeypatch.setattr(
|
|
gwmod.httpx, "AsyncClient", FakeAlwaysEmbeddedErrorClient_openai_gateway
|
|
)
|
|
svc = GatewayService()
|
|
cfg = svc.effective_config()
|
|
cfg["gateway_vision_enabled"] = False
|
|
cfg["gateway_max_retries"] = 0
|
|
rt = svc.runtime()
|
|
response = run_async(
|
|
rt.handle_chat(
|
|
{"messages": [{"role": "user", "content": "hi"}]},
|
|
cfg,
|
|
("guest", "embedded_error_no_fallback"),
|
|
"test",
|
|
"default",
|
|
)
|
|
)
|
|
assert response.status_code == 502
|
|
row = get_table("gateway_usage_ledger").find_one(owner_id="embedded_error_no_fallback")
|
|
assert row is not None
|
|
assert row["success"] == 0
|
|
assert row["error_category"] == "upstream_error"
|
|
|
|
|
|
def test_ollama_dialect_sends_reasoning_effort_alongside_think(local_db, monkeypatch):
|
|
# Ollama's native /api/chat honors a boolean `think`, but its
|
|
# OpenAI-compatible /v1/chat/completions layer (what this gateway
|
|
# actually calls) does not - it maps reasoning_effort/reasoning
|
|
# instead (github.com/ollama/ollama issues #15288, #15293, #14820).
|
|
monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeClient_openai_gateway)
|
|
svc = GatewayService()
|
|
cfg = svc.effective_config()
|
|
cfg["gateway_upstream_url"] = "http://127.0.0.1:11434/v1/chat/completions"
|
|
rt = svc.runtime()
|
|
run_async(
|
|
rt.handle_chat(
|
|
{"messages": [{"role": "user", "content": "hi"}]},
|
|
cfg,
|
|
("guest", "ollama_reasoning_effort"),
|
|
"test",
|
|
"default",
|
|
)
|
|
)
|
|
body = rt._client.calls[-1][1]
|
|
assert body["think"] is False
|
|
assert body["reasoning_effort"] == "none"
|
|
|
|
|
|
def test_hostile_upstream_model_header_is_ignored_end_to_end(local_db, monkeypatch):
|
|
monkeypatch.setattr(
|
|
gwmod.httpx, "AsyncClient", FakeClientHostileModelHeader_openai_gateway
|
|
)
|
|
svc = GatewayService()
|
|
cfg = svc.effective_config()
|
|
cfg["gateway_force_model"] = True
|
|
cfg["gateway_model"] = "deepseek-chat"
|
|
rt = svc.runtime()
|
|
resp = run_async(
|
|
rt.handle_chat(
|
|
{"messages": [{"role": "user", "content": "hi"}]},
|
|
cfg,
|
|
("guest", "hostile_header"),
|
|
"test",
|
|
"default",
|
|
)
|
|
)
|
|
assert resp.headers["X-Gateway-Model"] == "deepseek-chat"
|
|
|
|
|
|
def test_bypass_preamble_skips_the_operator_preamble_when_allowed(local_db, monkeypatch):
|
|
monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeClient_openai_gateway)
|
|
svc = GatewayService()
|
|
cfg = svc.effective_config()
|
|
cfg["gateway_system_preamble"] = "OPERATOR-PREAMBLE-TEXT"
|
|
rt = svc.runtime()
|
|
run_async(
|
|
rt.handle_chat(
|
|
{
|
|
"messages": [
|
|
{"role": "system", "content": "Reply with strict JSON only."},
|
|
{"role": "user", "content": "hi"},
|
|
],
|
|
"bypass_preamble": True,
|
|
},
|
|
cfg,
|
|
("internal", "devii"),
|
|
"test",
|
|
"default",
|
|
bypass_allowed=True,
|
|
)
|
|
)
|
|
sent = rt._client.calls[-1][1]
|
|
system_message = sent["messages"][0]
|
|
assert system_message["role"] == "system"
|
|
assert "OPERATOR-PREAMBLE-TEXT" not in system_message["content"]
|
|
assert "Current date" in system_message["content"]
|
|
assert "Reply with strict JSON only." in system_message["content"]
|
|
assert "bypass_preamble" not in sent
|
|
|
|
|
|
def test_bypass_preamble_ignored_without_internal_credentials(local_db, monkeypatch):
|
|
monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeClient_openai_gateway)
|
|
svc = GatewayService()
|
|
cfg = svc.effective_config()
|
|
cfg["gateway_system_preamble"] = "OPERATOR-PREAMBLE-TEXT"
|
|
rt = svc.runtime()
|
|
run_async(
|
|
rt.handle_chat(
|
|
{"messages": [{"role": "user", "content": "hi"}], "bypass_preamble": True},
|
|
cfg,
|
|
("user", "someuser"),
|
|
"test",
|
|
"default",
|
|
bypass_allowed=False,
|
|
)
|
|
)
|
|
sent = rt._client.calls[-1][1]
|
|
system_message = sent["messages"][0]
|
|
assert "OPERATOR-PREAMBLE-TEXT" in system_message["content"]
|
|
assert "bypass_preamble" not in sent
|
|
|
|
|
|
def test_bypass_preamble_keeps_the_clients_own_system_content(local_db, monkeypatch):
|
|
monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeClient_openai_gateway)
|
|
svc = GatewayService()
|
|
cfg = svc.effective_config()
|
|
cfg["gateway_system_preamble"] = "OPERATOR-PREAMBLE-TEXT"
|
|
rt = svc.runtime()
|
|
run_async(
|
|
rt.handle_chat(
|
|
{
|
|
"messages": [
|
|
{"role": "system", "content": "Grade strictly. Reply JSON only."},
|
|
{"role": "user", "content": "hi"},
|
|
],
|
|
"bypass_preamble": True,
|
|
},
|
|
cfg,
|
|
("internal", "devii"),
|
|
"test",
|
|
"default",
|
|
bypass_allowed=True,
|
|
)
|
|
)
|
|
sent = rt._client.calls[-1][1]
|
|
system_message = sent["messages"][0]
|
|
assert "OPERATOR-PREAMBLE-TEXT" not in system_message["content"]
|
|
assert "Grade strictly. Reply JSON only." in system_message["content"]
|
|
|
|
|
|
def test_internal_bypass_allowed_requires_the_exact_internal_key(local_db):
|
|
set_setting("gateway_internal_key", "test-internal-secret")
|
|
try:
|
|
svc = GatewayService()
|
|
cfg = svc.effective_config()
|
|
assert (
|
|
svc.internal_bypass_allowed(
|
|
_make_request_openai_gateway(
|
|
headers={"X-Gateway-Internal-Key": "test-internal-secret"}
|
|
),
|
|
cfg,
|
|
)
|
|
is True
|
|
)
|
|
assert (
|
|
svc.internal_bypass_allowed(
|
|
_make_request_openai_gateway(
|
|
headers={"X-Gateway-Internal-Key": "wrong-secret"}
|
|
),
|
|
cfg,
|
|
)
|
|
is False
|
|
)
|
|
assert (
|
|
svc.internal_bypass_allowed(
|
|
_make_request_openai_gateway(
|
|
headers={"Authorization": "Bearer test-internal-secret"}
|
|
),
|
|
cfg,
|
|
)
|
|
is False
|
|
)
|
|
assert svc.internal_bypass_allowed(_make_request_openai_gateway(), cfg) is False
|
|
finally:
|
|
set_setting("gateway_internal_key", "")
|
|
|
|
|
|
class FakeResponsesResp_openai_gateway(FakeResp_openai_gateway):
|
|
def __init__(self, status=200, payload=None, events=None, **kw):
|
|
super().__init__(status=status, payload=payload, **kw)
|
|
self._events = events or []
|
|
|
|
async def aiter_lines(self):
|
|
for event in self._events:
|
|
yield f"data: {json.dumps(event)}"
|
|
yield "data: [DONE]"
|
|
|
|
|
|
def _resp_obj_openai_gateway(model):
|
|
return {
|
|
"id": "resp_1",
|
|
"object": "response",
|
|
"created_at": 7,
|
|
"status": "completed",
|
|
"model": model,
|
|
"output": [
|
|
{
|
|
"type": "message",
|
|
"role": "assistant",
|
|
"content": [{"type": "output_text", "text": "hi there"}],
|
|
}
|
|
],
|
|
"usage": {"input_tokens": 5, "output_tokens": 7, "total_tokens": 12},
|
|
}
|
|
|
|
|
|
def _resp_events_openai_gateway(model):
|
|
return [
|
|
{"type": "response.created", "response": {"id": "resp_1"}},
|
|
{"type": "response.output_text.delta", "delta": "hi "},
|
|
{"type": "response.output_text.delta", "delta": "there"},
|
|
{"type": "response.completed", "response": _resp_obj_openai_gateway(model)},
|
|
]
|
|
|
|
|
|
def _resp_tool_obj_openai_gateway(model):
|
|
obj = _resp_obj_openai_gateway(model)
|
|
obj["output"].append(
|
|
{
|
|
"type": "function_call",
|
|
"call_id": "call_7",
|
|
"name": "ext_bash",
|
|
"arguments": "{}",
|
|
}
|
|
)
|
|
return obj
|
|
|
|
|
|
def _resp_tool_events_openai_gateway(model):
|
|
return [
|
|
{"type": "response.created", "response": {"id": "resp_1"}},
|
|
{
|
|
"type": "response.output_item.added",
|
|
"output_index": 1,
|
|
"item": {
|
|
"type": "function_call",
|
|
"call_id": "call_7",
|
|
"name": "ext_bash",
|
|
"arguments": "",
|
|
},
|
|
},
|
|
{
|
|
"type": "response.function_call_arguments.delta",
|
|
"item_id": "call_7",
|
|
"output_index": 1,
|
|
"delta": "{}",
|
|
},
|
|
{
|
|
"type": "response.completed",
|
|
"response": _resp_tool_obj_openai_gateway(model),
|
|
},
|
|
]
|
|
|
|
|
|
class FakeResponsesClient_openai_gateway:
|
|
behavior = "ok"
|
|
|
|
def __init__(self, *a, **k):
|
|
self.calls = []
|
|
self.last_headers = {}
|
|
|
|
def build_request(self, method, url, headers=None, json=None, content=None):
|
|
return FakeRequest(method, url, json, headers)
|
|
|
|
async def send(self, request, stream=False):
|
|
self.calls.append((request.url, request.json_body))
|
|
self.last_headers = dict(request.headers or {})
|
|
body = request.json_body or {}
|
|
if self.behavior == "error-envelope":
|
|
return FakeResp_openai_gateway(
|
|
status=429,
|
|
payload={
|
|
"error": {
|
|
"message": "slow down",
|
|
"type": "rate_limit",
|
|
"code": "rate_limit_exceeded",
|
|
}
|
|
},
|
|
)
|
|
if self.behavior == "failed-object":
|
|
return FakeResponsesResp_openai_gateway(
|
|
payload={
|
|
"object": "response",
|
|
"status": "failed",
|
|
"error": {"message": "boom"},
|
|
}
|
|
)
|
|
if self.behavior == "attestation-denied":
|
|
return FakeResp_openai_gateway(
|
|
status=403,
|
|
payload={
|
|
"error": {
|
|
"message": "This API key is billed to a Muse Code "
|
|
"subscription. Muse Code subscriptions cover requests "
|
|
"from the Muse Code client only.",
|
|
"type": "permission_error",
|
|
"code": "client_attestation_denied",
|
|
}
|
|
},
|
|
)
|
|
model = body.get("model", "")
|
|
if self.behavior == "tool-call":
|
|
return FakeResponsesResp_openai_gateway(
|
|
payload=_resp_tool_obj_openai_gateway(model),
|
|
events=_resp_tool_events_openai_gateway(model),
|
|
)
|
|
return FakeResponsesResp_openai_gateway(
|
|
payload=_resp_obj_openai_gateway(model),
|
|
events=_resp_events_openai_gateway(model),
|
|
)
|
|
|
|
async def post(self, url, headers=None, json=None, timeout=None):
|
|
return await self.send(FakeRequest("POST", url, json), stream=False)
|
|
|
|
async def aclose(self):
|
|
pass
|
|
|
|
|
|
def _setup_responses_route_openai_gateway():
|
|
from devplacepy.services.openai_gateway import routing
|
|
|
|
routing.provider_store.set(
|
|
routing.ProviderIn(
|
|
name="gwmeta",
|
|
base_url="https://api.meta.ai/v1/responses",
|
|
api_key="sk-test",
|
|
protocol="responses",
|
|
)
|
|
)
|
|
routing.model_store.set(
|
|
routing.ModelRouteIn(
|
|
source_model="meta-chat",
|
|
provider="gwmeta",
|
|
target_model="muse-spark-1.3-contributor",
|
|
kind="chat",
|
|
)
|
|
)
|
|
|
|
|
|
def _teardown_responses_route_openai_gateway():
|
|
from devplacepy.services.openai_gateway import routing
|
|
|
|
routing.model_store.remove("meta-chat")
|
|
routing.provider_store.remove("gwmeta")
|
|
|
|
|
|
def _chat_body_openai_gateway(**over):
|
|
body = {
|
|
"model": "meta-chat",
|
|
"messages": [
|
|
{"role": "system", "content": "Be terse."},
|
|
{"role": "user", "content": "hi"},
|
|
],
|
|
}
|
|
body.update(over)
|
|
return body
|
|
|
|
|
|
def test_responses_protocol_transpiles_request_and_response(local_db, monkeypatch):
|
|
monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeResponsesClient_openai_gateway)
|
|
FakeResponsesClient_openai_gateway.behavior = "ok"
|
|
_setup_responses_route_openai_gateway()
|
|
try:
|
|
svc = GatewayService()
|
|
cfg = svc.effective_config()
|
|
cfg["gateway_vision_enabled"] = False
|
|
rt = svc.runtime()
|
|
resp = run_async(
|
|
rt.handle_chat(
|
|
_chat_body_openai_gateway(), cfg, ("guest", "test"), "test", "default"
|
|
)
|
|
)
|
|
url, sent = rt._client.calls[-1]
|
|
assert str(url) == "https://api.meta.ai/v1/responses"
|
|
assert sent["model"] == "muse-spark-1.3-contributor"
|
|
assert "messages" not in sent
|
|
assert "Be terse." in sent["instructions"]
|
|
assert sent["input"] == [
|
|
{"type": "message", "role": "user", "content": "hi"}
|
|
]
|
|
assert sent["reasoning"] == {"effort": "minimal"}
|
|
assert resp.status_code == 200
|
|
payload = json.loads(bytes(resp.body).decode())
|
|
assert payload["object"] == "chat.completion"
|
|
assert payload["choices"][0]["message"]["content"] == "hi there"
|
|
assert payload["choices"][0]["finish_reason"] == "stop"
|
|
assert payload["usage"]["prompt_tokens"] == 5
|
|
assert resp.headers["X-Gateway-Model"] == "muse-spark-1.3-contributor"
|
|
finally:
|
|
_teardown_responses_route_openai_gateway()
|
|
|
|
|
|
def test_muse_profile_headers_reach_the_wire(local_db, monkeypatch):
|
|
from devplacepy.services.openai_gateway import routing
|
|
|
|
monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeResponsesClient_openai_gateway)
|
|
FakeResponsesClient_openai_gateway.behavior = "ok"
|
|
routing.provider_store.set(
|
|
routing.ProviderIn(
|
|
name="wiremeta",
|
|
base_url="https://api.meta.ai/v1/responses",
|
|
api_key="sk-test",
|
|
protocol="responses",
|
|
client_profile="muse",
|
|
)
|
|
)
|
|
routing.model_store.set(
|
|
routing.ModelRouteIn(
|
|
source_model="wire-muse",
|
|
provider="wiremeta",
|
|
target_model="muse-spark",
|
|
kind="chat",
|
|
)
|
|
)
|
|
try:
|
|
svc = GatewayService()
|
|
cfg = svc.effective_config()
|
|
cfg["gateway_vision_enabled"] = False
|
|
rt = svc.runtime()
|
|
logged = []
|
|
resp = run_async(
|
|
rt.handle_chat(
|
|
{"model": "wire-muse", "messages": [{"role": "user", "content": "hi"}]},
|
|
cfg,
|
|
("guest", "test"),
|
|
"test",
|
|
"default",
|
|
log=logged.append,
|
|
)
|
|
)
|
|
assert resp.status_code == 200
|
|
sent_headers = rt._client.last_headers
|
|
assert sent_headers["X-Client-Id"] == "tbh:exec"
|
|
assert sent_headers["User-Agent"].startswith("muse-build/")
|
|
session = sent_headers["X-Tbh-Session-Id"]
|
|
assert session
|
|
assert sent_headers["X-Meta-Ai-Gateway-Session-Id"] == session
|
|
assert sent_headers["X-Tbh-Thread-Id"] == session
|
|
assert sent_headers["X-Tbh-Root-Session-Id"] == session
|
|
assert any("(client profile 'muse')" in line for line in logged)
|
|
finally:
|
|
routing.model_store.remove("wire-muse")
|
|
routing.provider_store.remove("wiremeta")
|
|
|
|
|
|
def _bash_tool_body_openai_gateway(**over):
|
|
body = _chat_body_openai_gateway(**over)
|
|
body["tools"] = [
|
|
{
|
|
"type": "function",
|
|
"function": {
|
|
"name": "bash",
|
|
"description": "run shell",
|
|
"parameters": {"type": "object", "properties": {}},
|
|
},
|
|
}
|
|
]
|
|
return body
|
|
|
|
|
|
def test_meta_tool_names_aliased_on_wire_and_restored(local_db, monkeypatch):
|
|
monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeResponsesClient_openai_gateway)
|
|
FakeResponsesClient_openai_gateway.behavior = "tool-call"
|
|
_setup_responses_route_openai_gateway()
|
|
try:
|
|
svc = GatewayService()
|
|
cfg = svc.effective_config()
|
|
cfg["gateway_vision_enabled"] = False
|
|
rt = svc.runtime()
|
|
logged = []
|
|
resp = run_async(
|
|
rt.handle_chat(
|
|
_bash_tool_body_openai_gateway(),
|
|
cfg,
|
|
("guest", "test"),
|
|
"test",
|
|
"default",
|
|
log=logged.append,
|
|
)
|
|
)
|
|
_, sent = rt._client.calls[-1]
|
|
assert sent["tools"][0]["name"] == "ext_bash"
|
|
assert any("bash->ext_bash" in line for line in logged)
|
|
assert resp.status_code == 200
|
|
payload = json.loads(bytes(resp.body).decode())
|
|
calls = payload["choices"][0]["message"]["tool_calls"]
|
|
assert calls[0]["function"]["name"] == "bash"
|
|
assert payload["choices"][0]["finish_reason"] == "tool_calls"
|
|
finally:
|
|
FakeResponsesClient_openai_gateway.behavior = "ok"
|
|
_teardown_responses_route_openai_gateway()
|
|
|
|
|
|
def test_meta_tool_names_restored_on_stream_chunks(local_db, monkeypatch):
|
|
monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeResponsesClient_openai_gateway)
|
|
FakeResponsesClient_openai_gateway.behavior = "tool-call"
|
|
_setup_responses_route_openai_gateway()
|
|
try:
|
|
svc = GatewayService()
|
|
cfg = svc.effective_config()
|
|
cfg["gateway_vision_enabled"] = False
|
|
rt = svc.runtime()
|
|
resp = run_async(
|
|
rt.handle_chat(
|
|
_bash_tool_body_openai_gateway(stream=True),
|
|
cfg,
|
|
("guest", "test"),
|
|
"test",
|
|
"default",
|
|
)
|
|
)
|
|
|
|
async def drain():
|
|
out = []
|
|
async for chunk in resp.body_iterator:
|
|
out.append(chunk if isinstance(chunk, str) else chunk.decode())
|
|
return "".join(out)
|
|
|
|
body = run_async(drain())
|
|
assert '"name": "bash"' in body
|
|
assert "ext_bash" not in body
|
|
assert "[DONE]" in body
|
|
finally:
|
|
FakeResponsesClient_openai_gateway.behavior = "ok"
|
|
_teardown_responses_route_openai_gateway()
|
|
|
|
|
|
def _setup_chat_meta_route_openai_gateway():
|
|
from devplacepy.services.openai_gateway import routing
|
|
|
|
routing.provider_store.set(
|
|
routing.ProviderIn(
|
|
name="gwmetacc",
|
|
base_url="https://api.meta.ai/v1/chat/completions",
|
|
api_key="sk-test",
|
|
)
|
|
)
|
|
routing.model_store.set(
|
|
routing.ModelRouteIn(
|
|
source_model="meta-chat-cc",
|
|
provider="gwmetacc",
|
|
target_model="muse-spark",
|
|
kind="chat",
|
|
)
|
|
)
|
|
|
|
|
|
def _teardown_chat_meta_route_openai_gateway():
|
|
from devplacepy.services.openai_gateway import routing
|
|
|
|
routing.model_store.remove("meta-chat-cc")
|
|
routing.provider_store.remove("gwmetacc")
|
|
|
|
|
|
def _tai_tool_body_openai_gateway(**over):
|
|
# Mirrors a real tai/detz turn: core tools are always sent, so "shell"
|
|
# (a Pi-verified gate trigger missing from the denylist until
|
|
# 2026-11-10) rides every request alongside the lazy "read_file".
|
|
body = _chat_body_openai_gateway(**over)
|
|
body["model"] = "meta-chat-cc"
|
|
body["tools"] = [
|
|
{
|
|
"type": "function",
|
|
"function": {
|
|
"name": "shell",
|
|
"description": "run a shell command",
|
|
"parameters": {"type": "object", "properties": {}},
|
|
},
|
|
},
|
|
{
|
|
"type": "function",
|
|
"function": {
|
|
"name": "read_file",
|
|
"description": "read a file",
|
|
"parameters": {"type": "object", "properties": {}},
|
|
},
|
|
},
|
|
{
|
|
"type": "function",
|
|
"function": {
|
|
"name": "web_fetch",
|
|
"description": "fetch a URL",
|
|
"parameters": {"type": "object", "properties": {}},
|
|
},
|
|
},
|
|
]
|
|
return body
|
|
|
|
|
|
class FakeChatMetaStreamResp_openai_gateway(FakeResp_openai_gateway):
|
|
def __init__(self, payload):
|
|
super().__init__(status=200, payload=payload)
|
|
self._payload = payload
|
|
|
|
async def aiter_lines(self):
|
|
calls = self._payload["choices"][0]["message"]["tool_calls"]
|
|
yield (
|
|
'data: {"id":"chatcmpl-x","object":"chat.completion.chunk",'
|
|
'"model":"m","choices":[{"index":0,"delta":{"role":"assistant"}}]}'
|
|
)
|
|
for pos, call in enumerate(calls):
|
|
name = call["function"]["name"]
|
|
yield (
|
|
'data: {"id":"chatcmpl-x","object":"chat.completion.chunk",'
|
|
'"model":"m","choices":[{"index":0,"delta":{"tool_calls":'
|
|
'[{"index":%d,"id":"call_%d","function":{"name":"%s",'
|
|
'"arguments":"{}"}}]}}]}' % (pos, pos, name)
|
|
)
|
|
yield (
|
|
'data: {"id":"chatcmpl-x","object":"chat.completion.chunk",'
|
|
'"model":"m","choices":[{"index":0,"delta":{},'
|
|
'"finish_reason":"tool_calls"}]}'
|
|
)
|
|
yield "data: [DONE]"
|
|
|
|
|
|
class FakeChatMetaClient_openai_gateway:
|
|
def __init__(self, *a, **k):
|
|
self.calls = []
|
|
|
|
def build_request(self, method, url, headers=None, json=None, content=None):
|
|
return FakeRequest(method, url, json, headers)
|
|
|
|
async def send(self, request, stream=False):
|
|
self.calls.append((request.url, request.json_body))
|
|
body = request.json_body or {}
|
|
wire_names = [
|
|
(tool.get("function") or {}).get("name")
|
|
for tool in body.get("tools") or []
|
|
]
|
|
payload = {
|
|
"id": "chatcmpl-x",
|
|
"object": "chat.completion",
|
|
"model": body.get("model"),
|
|
"choices": [
|
|
{
|
|
"index": 0,
|
|
"message": {
|
|
"role": "assistant",
|
|
"content": "",
|
|
"tool_calls": [
|
|
{
|
|
"id": "call_7",
|
|
"type": "function",
|
|
"function": {"name": name, "arguments": "{}"},
|
|
}
|
|
for name in wire_names
|
|
],
|
|
},
|
|
"finish_reason": "tool_calls",
|
|
}
|
|
],
|
|
"usage": {"prompt_tokens": 5, "completion_tokens": 2, "total_tokens": 7},
|
|
}
|
|
if stream:
|
|
return FakeChatMetaStreamResp_openai_gateway(payload)
|
|
return FakeResp_openai_gateway(
|
|
payload=payload, content=json.dumps(payload).encode()
|
|
)
|
|
|
|
async def post(self, url, headers=None, json=None, timeout=None):
|
|
return await self.send(FakeRequest("POST", url, json), stream=False)
|
|
|
|
async def aclose(self):
|
|
pass
|
|
|
|
|
|
def test_meta_tool_names_aliased_on_chat_completions_wire(local_db, monkeypatch):
|
|
monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeChatMetaClient_openai_gateway)
|
|
_setup_chat_meta_route_openai_gateway()
|
|
try:
|
|
svc = GatewayService()
|
|
cfg = svc.effective_config()
|
|
cfg["gateway_vision_enabled"] = False
|
|
rt = svc.runtime()
|
|
logged = []
|
|
resp = run_async(
|
|
rt.handle_chat(
|
|
_tai_tool_body_openai_gateway(),
|
|
cfg,
|
|
("guest", "test"),
|
|
"test",
|
|
"default",
|
|
log=logged.append,
|
|
)
|
|
)
|
|
_, sent = rt._client.calls[-1]
|
|
wire = [tool["function"]["name"] for tool in sent["tools"]]
|
|
assert wire == ["ext_shell", "ext_read_file", "ext_web_fetch"]
|
|
assert any("shell->ext_shell" in line for line in logged)
|
|
assert any("read_file->ext_read_file" in line for line in logged)
|
|
assert any("web_fetch->ext_web_fetch" in line for line in logged)
|
|
assert resp.status_code == 200
|
|
payload = json.loads(bytes(resp.body).decode())
|
|
calls = payload["choices"][0]["message"]["tool_calls"]
|
|
assert [call["function"]["name"] for call in calls] == [
|
|
"shell",
|
|
"read_file",
|
|
"web_fetch",
|
|
]
|
|
assert payload["choices"][0]["finish_reason"] == "tool_calls"
|
|
finally:
|
|
_teardown_chat_meta_route_openai_gateway()
|
|
|
|
|
|
def test_meta_tool_names_restored_on_chat_completions_stream(
|
|
local_db, monkeypatch
|
|
):
|
|
monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeChatMetaClient_openai_gateway)
|
|
_setup_chat_meta_route_openai_gateway()
|
|
try:
|
|
svc = GatewayService()
|
|
cfg = svc.effective_config()
|
|
cfg["gateway_vision_enabled"] = False
|
|
rt = svc.runtime()
|
|
resp = run_async(
|
|
rt.handle_chat(
|
|
_tai_tool_body_openai_gateway(stream=True),
|
|
cfg,
|
|
("guest", "test"),
|
|
"test",
|
|
"default",
|
|
)
|
|
)
|
|
|
|
async def drain():
|
|
out = []
|
|
async for chunk in resp.body_iterator:
|
|
out.append(chunk if isinstance(chunk, str) else chunk.decode())
|
|
return "".join(out)
|
|
|
|
body = run_async(drain())
|
|
_, sent = rt._client.calls[-1]
|
|
wire = [tool["function"]["name"] for tool in sent["tools"]]
|
|
assert wire == ["ext_shell", "ext_read_file", "ext_web_fetch"]
|
|
assert '"name": "shell"' in body
|
|
assert '"name": "read_file"' in body
|
|
assert '"name": "web_fetch"' in body
|
|
assert "ext_shell" not in body
|
|
assert "ext_read_file" not in body
|
|
assert "ext_web_fetch" not in body
|
|
assert "[DONE]" in body
|
|
finally:
|
|
_teardown_chat_meta_route_openai_gateway()
|
|
|
|
|
|
def test_responses_protocol_stream_relays_chat_chunks(local_db, monkeypatch):
|
|
monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeResponsesClient_openai_gateway)
|
|
FakeResponsesClient_openai_gateway.behavior = "ok"
|
|
_setup_responses_route_openai_gateway()
|
|
try:
|
|
svc = GatewayService()
|
|
cfg = svc.effective_config()
|
|
cfg["gateway_vision_enabled"] = False
|
|
rt = svc.runtime()
|
|
resp = run_async(
|
|
rt.handle_chat(
|
|
_chat_body_openai_gateway(stream=True),
|
|
cfg,
|
|
("guest", "test"),
|
|
"test",
|
|
"default",
|
|
)
|
|
)
|
|
|
|
async def drain():
|
|
out = []
|
|
async for chunk in resp.body_iterator:
|
|
out.append(chunk if isinstance(chunk, str) else chunk.decode())
|
|
return "".join(out)
|
|
|
|
body = run_async(drain())
|
|
assert "chat.completion.chunk" in body
|
|
assert "hi " in body and "there" in body
|
|
assert '"usage"' not in body
|
|
assert "[DONE]" in body
|
|
finally:
|
|
_teardown_responses_route_openai_gateway()
|
|
|
|
|
|
def test_responses_protocol_stream_relays_usage_when_asked(local_db, monkeypatch):
|
|
monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeResponsesClient_openai_gateway)
|
|
FakeResponsesClient_openai_gateway.behavior = "ok"
|
|
_setup_responses_route_openai_gateway()
|
|
try:
|
|
svc = GatewayService()
|
|
cfg = svc.effective_config()
|
|
cfg["gateway_vision_enabled"] = False
|
|
rt = svc.runtime()
|
|
resp = run_async(
|
|
rt.handle_chat(
|
|
_chat_body_openai_gateway(
|
|
stream=True, stream_options={"include_usage": True}
|
|
),
|
|
cfg,
|
|
("guest", "test"),
|
|
"test",
|
|
"default",
|
|
)
|
|
)
|
|
_, sent = rt._client.calls[-1]
|
|
assert "stream_options" not in sent
|
|
|
|
async def drain():
|
|
out = []
|
|
async for chunk in resp.body_iterator:
|
|
out.append(chunk if isinstance(chunk, str) else chunk.decode())
|
|
return "".join(out)
|
|
|
|
body = run_async(drain())
|
|
assert '"usage"' in body
|
|
assert '"prompt_tokens": 5' in body
|
|
usage_chunks = [
|
|
json.loads(line[len("data:"):])
|
|
for line in body.splitlines()
|
|
if line.startswith("data: {") and '"usage"' in line
|
|
]
|
|
assert len(usage_chunks) == 1
|
|
assert usage_chunks[0]["choices"] == []
|
|
finally:
|
|
_teardown_responses_route_openai_gateway()
|
|
|
|
|
|
def test_responses_protocol_relays_error_envelope(local_db, monkeypatch):
|
|
monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeResponsesClient_openai_gateway)
|
|
FakeResponsesClient_openai_gateway.behavior = "error-envelope"
|
|
_setup_responses_route_openai_gateway()
|
|
try:
|
|
svc = GatewayService()
|
|
cfg = svc.effective_config()
|
|
cfg["gateway_vision_enabled"] = False
|
|
rt = svc.runtime()
|
|
resp = run_async(
|
|
rt.handle_chat(
|
|
_chat_body_openai_gateway(), cfg, ("guest", "test"), "test", "default"
|
|
)
|
|
)
|
|
assert resp.status_code == 429
|
|
payload = json.loads(bytes(resp.body).decode())
|
|
assert payload["error"]["message"] == "slow down"
|
|
assert payload["error"]["code"] == "rate_limit_exceeded"
|
|
finally:
|
|
FakeResponsesClient_openai_gateway.behavior = "ok"
|
|
_teardown_responses_route_openai_gateway()
|
|
|
|
|
|
def test_responses_protocol_attestation_denied_logs_guidance(
|
|
local_db, monkeypatch
|
|
):
|
|
monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeResponsesClient_openai_gateway)
|
|
FakeResponsesClient_openai_gateway.behavior = "attestation-denied"
|
|
_setup_responses_route_openai_gateway()
|
|
try:
|
|
svc = GatewayService()
|
|
cfg = svc.effective_config()
|
|
cfg["gateway_vision_enabled"] = False
|
|
rt = svc.runtime()
|
|
logged = []
|
|
resp = run_async(
|
|
rt.handle_chat(
|
|
_chat_body_openai_gateway(),
|
|
cfg,
|
|
("guest", "test"),
|
|
"test",
|
|
"default",
|
|
log=logged.append,
|
|
)
|
|
)
|
|
assert resp.status_code == 403
|
|
payload = json.loads(bytes(resp.body).decode())
|
|
assert payload["error"]["code"] == "client_attestation_denied"
|
|
assert any("pay-as-you-go" in line for line in logged)
|
|
finally:
|
|
FakeResponsesClient_openai_gateway.behavior = "ok"
|
|
_teardown_responses_route_openai_gateway()
|
|
|
|
|
|
def test_upstream_auth_guidance_covers_401_and_403():
|
|
denied = FakeResp_openai_gateway(
|
|
status=403,
|
|
payload={
|
|
"error": {
|
|
"message": "subscription only",
|
|
"type": "permission_error",
|
|
"code": "client_attestation_denied",
|
|
}
|
|
},
|
|
)
|
|
guidance = gwmod._upstream_auth_guidance(denied, denied.json())
|
|
assert "pay-as-you-go" in guidance
|
|
assert "aliasing" in guidance
|
|
assert "MUSE_RESERVED_TOOL_NAMES" in guidance
|
|
other_denied = FakeResp_openai_gateway(status=403, payload={"error": "nope"})
|
|
assert (
|
|
gwmod._upstream_auth_guidance(other_denied, other_denied.json()) is None
|
|
)
|
|
unauthed = FakeResp_openai_gateway(
|
|
status=401, payload={"error": {"message": "User not found.", "code": 401}}
|
|
)
|
|
guidance = gwmod._upstream_auth_guidance(unauthed, unauthed.json())
|
|
assert "api_key" in guidance
|
|
assert "User not found." in guidance
|
|
server_error = FakeResp_openai_gateway(status=500, payload={"error": "x"})
|
|
assert (
|
|
gwmod._upstream_auth_guidance(server_error, server_error.json()) is None
|
|
)
|
|
|
|
|
|
def test_should_fallback_model_unavailable_but_not_bad_request():
|
|
unavailable = FakeResp_openai_gateway(
|
|
status=400,
|
|
payload={"error": {"message": "Model is unavailable."}},
|
|
)
|
|
assert (
|
|
gwmod._should_fallback(unavailable, None, {"circuit_open": False}) is True
|
|
)
|
|
malformed = FakeResp_openai_gateway(
|
|
status=400, payload={"error": "bad schema"}
|
|
)
|
|
assert (
|
|
gwmod._should_fallback(malformed, None, {"circuit_open": False}) is False
|
|
)
|
|
|
|
|
|
def test_responses_protocol_failed_object_is_502(local_db, monkeypatch):
|
|
monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeResponsesClient_openai_gateway)
|
|
FakeResponsesClient_openai_gateway.behavior = "failed-object"
|
|
_setup_responses_route_openai_gateway()
|
|
try:
|
|
svc = GatewayService()
|
|
cfg = svc.effective_config()
|
|
cfg["gateway_vision_enabled"] = False
|
|
rt = svc.runtime()
|
|
resp = run_async(
|
|
rt.handle_chat(
|
|
_chat_body_openai_gateway(), cfg, ("guest", "test"), "test", "default"
|
|
)
|
|
)
|
|
assert resp.status_code == 502
|
|
finally:
|
|
FakeResponsesClient_openai_gateway.behavior = "ok"
|
|
_teardown_responses_route_openai_gateway()
|
|
|
|
|
|
def test_responses_protocol_client_reasoning_effort_passes_through(
|
|
local_db, monkeypatch
|
|
):
|
|
monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeResponsesClient_openai_gateway)
|
|
FakeResponsesClient_openai_gateway.behavior = "ok"
|
|
_setup_responses_route_openai_gateway()
|
|
try:
|
|
svc = GatewayService()
|
|
cfg = svc.effective_config()
|
|
cfg["gateway_vision_enabled"] = False
|
|
rt = svc.runtime()
|
|
run_async(
|
|
rt.handle_chat(
|
|
_chat_body_openai_gateway(reasoning_effort="high"),
|
|
cfg,
|
|
("guest", "test"),
|
|
"test",
|
|
"default",
|
|
)
|
|
)
|
|
_, sent = rt._client.calls[-1]
|
|
assert sent["reasoning"] == {"effort": "high"}
|
|
finally:
|
|
_teardown_responses_route_openai_gateway()
|
|
|
|
|
|
def test_responses_protocol_default_provider(local_db, monkeypatch):
|
|
monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeResponsesClient_openai_gateway)
|
|
FakeResponsesClient_openai_gateway.behavior = "ok"
|
|
svc = GatewayService()
|
|
cfg = svc.effective_config()
|
|
cfg["gateway_vision_enabled"] = False
|
|
cfg["gateway_upstream_url"] = "https://api.meta.ai/v1/responses"
|
|
cfg["gateway_upstream_protocol"] = "responses"
|
|
cfg["gateway_model"] = "muse-spark-1.3-contributor"
|
|
rt = svc.runtime()
|
|
resp = run_async(
|
|
rt.handle_chat(
|
|
{"messages": [{"role": "user", "content": "hi"}]},
|
|
cfg,
|
|
("guest", "test"),
|
|
"test",
|
|
"default",
|
|
)
|
|
)
|
|
url, sent = rt._client.calls[-1]
|
|
assert str(url) == "https://api.meta.ai/v1/responses"
|
|
assert sent["input"] == [{"type": "message", "role": "user", "content": "hi"}]
|
|
assert resp.status_code == 200
|
|
payload = json.loads(bytes(resp.body).decode())
|
|
assert payload["object"] == "chat.completion"
|
|
|
|
|
|
EMULATED_CALL_TEXT_openai_gateway = (
|
|
"Sure.\n<tool_call>\n"
|
|
'{"name": "bash", "arguments": {"command": "ls"}}\n'
|
|
"</tool_call>"
|
|
)
|
|
BASH_TOOL_openai_gateway = {
|
|
"type": "function",
|
|
"function": {
|
|
"name": "bash",
|
|
"description": "Run a shell command",
|
|
"parameters": {
|
|
"type": "object",
|
|
"properties": {"command": {"type": "string"}},
|
|
"required": ["command"],
|
|
},
|
|
},
|
|
}
|
|
|
|
|
|
def _emulated_obj_openai_gateway(model, text):
|
|
obj = _resp_obj_openai_gateway(model)
|
|
obj["output"][0]["content"][0]["text"] = text
|
|
return obj
|
|
|
|
|
|
class FakeEmulatedClient_openai_gateway(FakeResponsesClient_openai_gateway):
|
|
text = EMULATED_CALL_TEXT_openai_gateway
|
|
|
|
async def send(self, request, stream=False):
|
|
self.calls.append((request.url, request.json_body))
|
|
model = (request.json_body or {}).get("model", "")
|
|
obj = _emulated_obj_openai_gateway(model, self.text)
|
|
events = [{"type": "response.created", "response": {"id": "resp_1"}}]
|
|
for i in range(0, len(self.text), 4):
|
|
events.append(
|
|
{"type": "response.output_text.delta", "delta": self.text[i : i + 4]}
|
|
)
|
|
events.append({"type": "response.completed", "response": obj})
|
|
return FakeResponsesResp_openai_gateway(payload=obj, events=events)
|
|
|
|
|
|
def _setup_emulated_route_openai_gateway():
|
|
from devplacepy.services.openai_gateway import routing
|
|
|
|
routing.provider_store.set(
|
|
routing.ProviderIn(
|
|
name="gwmeta",
|
|
base_url="https://api.meta.ai/v1/responses",
|
|
api_key="sk-test",
|
|
protocol="responses",
|
|
)
|
|
)
|
|
routing.model_store.set(
|
|
routing.ModelRouteIn(
|
|
source_model="meta-chat",
|
|
provider="gwmeta",
|
|
target_model="muse-spark-1.3-contributor",
|
|
kind="chat",
|
|
tools_mode="emulated",
|
|
)
|
|
)
|
|
|
|
|
|
def _emulated_chat_openai_gateway(monkeypatch, body, **tools):
|
|
monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeEmulatedClient_openai_gateway)
|
|
_setup_emulated_route_openai_gateway()
|
|
svc = GatewayService()
|
|
cfg = svc.effective_config()
|
|
cfg["gateway_vision_enabled"] = False
|
|
rt = svc.runtime()
|
|
resp = run_async(rt.handle_chat(body, cfg, ("guest", "test"), "test", "default"))
|
|
return rt, resp
|
|
|
|
|
|
def test_emulated_tools_meta_responses_nonstream(local_db, monkeypatch):
|
|
body = _chat_body_openai_gateway(tools=[BASH_TOOL_openai_gateway], tool_choice="auto")
|
|
try:
|
|
rt, resp = _emulated_chat_openai_gateway(monkeypatch, body)
|
|
url, sent = rt._client.calls[-1]
|
|
assert str(url) == "https://api.meta.ai/v1/responses"
|
|
assert "tools" not in sent and "tool_choice" not in sent
|
|
assert "<tools>" in sent["instructions"] and '"bash"' in sent["instructions"]
|
|
assert resp.status_code == 200
|
|
message = json.loads(bytes(resp.body).decode())["choices"][0]
|
|
assert message["finish_reason"] == "tool_calls"
|
|
call = message["message"]["tool_calls"][0]
|
|
assert call["function"]["name"] == "bash"
|
|
assert json.loads(call["function"]["arguments"]) == {"command": "ls"}
|
|
assert message["message"]["content"] == "Sure."
|
|
finally:
|
|
_teardown_responses_route_openai_gateway()
|
|
|
|
|
|
def test_emulated_tools_meta_responses_stream(local_db, monkeypatch):
|
|
body = _chat_body_openai_gateway(
|
|
stream=True, tools=[BASH_TOOL_openai_gateway]
|
|
)
|
|
try:
|
|
rt, resp = _emulated_chat_openai_gateway(monkeypatch, body)
|
|
|
|
async def drain():
|
|
out = []
|
|
async for chunk in resp.body_iterator:
|
|
out.append(chunk if isinstance(chunk, str) else chunk.decode())
|
|
return "".join(out)
|
|
|
|
raw = run_async(drain())
|
|
chunks = [
|
|
json.loads(line[5:])
|
|
for line in raw.splitlines()
|
|
if line.startswith("data:") and "[DONE]" not in line
|
|
]
|
|
text = "".join(
|
|
c["choices"][0]["delta"].get("content", "")
|
|
for c in chunks
|
|
if c.get("choices")
|
|
)
|
|
calls = [
|
|
tc
|
|
for c in chunks
|
|
for choice in c.get("choices") or []
|
|
for tc in choice["delta"].get("tool_calls") or []
|
|
]
|
|
assert "<tool_call>" not in text and text.strip() == "Sure."
|
|
assert len(calls) == 1 and calls[0]["function"]["name"] == "bash"
|
|
finishes = [
|
|
c["choices"][0]["finish_reason"]
|
|
for c in chunks
|
|
if c.get("choices") and c["choices"][0].get("finish_reason")
|
|
]
|
|
assert finishes[-1] == "tool_calls"
|
|
assert raw.rstrip().endswith("[DONE]")
|
|
finally:
|
|
_teardown_responses_route_openai_gateway()
|
|
|
|
|
|
def test_emulated_tools_convert_history_for_the_meta_transpiler(local_db, monkeypatch):
|
|
history = [
|
|
{"role": "user", "content": "go"},
|
|
{
|
|
"role": "assistant",
|
|
"content": None,
|
|
"tool_calls": [
|
|
{
|
|
"id": "call_1",
|
|
"type": "function",
|
|
"function": {"name": "bash", "arguments": '{"command": "ls"}'},
|
|
}
|
|
],
|
|
},
|
|
{"role": "tool", "tool_call_id": "call_1", "content": "a.txt"},
|
|
]
|
|
body = _chat_body_openai_gateway(messages=history, tools=[BASH_TOOL_openai_gateway])
|
|
try:
|
|
rt, resp = _emulated_chat_openai_gateway(monkeypatch, body)
|
|
_, sent = rt._client.calls[-1]
|
|
assert resp.status_code == 200
|
|
assert all(item.get("type") == "message" for item in sent["input"])
|
|
joined = json.dumps(sent["input"])
|
|
assert "<tool_call>" in joined and "<tool_response>" in joined and "a.txt" in joined
|
|
assert "function_call" not in joined
|
|
finally:
|
|
_teardown_responses_route_openai_gateway()
|
|
|
|
|
|
def test_emulated_tools_serve_large_catalogs_with_no_cap(local_db, monkeypatch):
|
|
tools = [
|
|
{
|
|
"type": "function",
|
|
"function": {
|
|
"name": f"tool_{index}",
|
|
"description": f"tool number {index}",
|
|
"parameters": {"type": "object", "properties": {}, "required": []},
|
|
},
|
|
}
|
|
for index in range(200)
|
|
]
|
|
body = _chat_body_openai_gateway(tools=tools, tool_choice="auto")
|
|
try:
|
|
rt, resp = _emulated_chat_openai_gateway(monkeypatch, body)
|
|
assert resp.status_code == 200
|
|
assert len(rt._client.calls) == 1
|
|
_, sent = rt._client.calls[-1]
|
|
assert "tools" not in sent and "tool_choice" not in sent
|
|
assert '"name": "tool_0"' in sent["instructions"]
|
|
assert '"name": "tool_199"' in sent["instructions"]
|
|
assert len(rt._catalog_cache._entries) == 1
|
|
finally:
|
|
_teardown_responses_route_openai_gateway()
|
|
|
|
|
|
def test_native_route_keeps_forwarding_tools(local_db, monkeypatch):
|
|
monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeResponsesClient_openai_gateway)
|
|
FakeResponsesClient_openai_gateway.behavior = "ok"
|
|
_setup_responses_route_openai_gateway()
|
|
try:
|
|
svc = GatewayService()
|
|
cfg = svc.effective_config()
|
|
cfg["gateway_vision_enabled"] = False
|
|
rt = svc.runtime()
|
|
run_async(
|
|
rt.handle_chat(
|
|
_chat_body_openai_gateway(tools=[BASH_TOOL_openai_gateway]),
|
|
cfg,
|
|
("guest", "test"),
|
|
"test",
|
|
"default",
|
|
)
|
|
)
|
|
_, sent = rt._client.calls[-1]
|
|
assert "tools" in sent
|
|
assert "<tools>" not in sent["instructions"]
|
|
finally:
|
|
_teardown_responses_route_openai_gateway()
|
|
|
|
|
|
VAIDS_REJECTION_openai_gateway = {
|
|
"details": {"_errors": ["Unrecognized key(s) in object: 'thinking'"]},
|
|
"error": "Invalid request parameters",
|
|
"issues": [
|
|
{
|
|
"code": "unrecognized_keys",
|
|
"keys": ["thinking"],
|
|
"path": [],
|
|
"message": "Unrecognized key(s) in object: 'thinking'",
|
|
}
|
|
],
|
|
}
|
|
|
|
|
|
def _rejection_openai_gateway(key):
|
|
return {
|
|
"details": {"_errors": [f"Unrecognized key(s) in object: '{key}'"]},
|
|
"error": "Invalid request parameters",
|
|
"issues": [{"code": "unrecognized_keys", "keys": [key], "path": []}],
|
|
}
|
|
|
|
|
|
class FakeStrictClient_openai_gateway:
|
|
rejected = ("thinking",)
|
|
|
|
def __init__(self, *a, **k):
|
|
self.calls = []
|
|
|
|
def build_request(self, method, url, headers=None, json=None, content=None):
|
|
return FakeRequest(method, url, json)
|
|
|
|
async def send(self, request, stream=False):
|
|
body = request.json_body or {}
|
|
self.calls.append((request.url, dict(body)))
|
|
for key in self.rejected:
|
|
if key in body:
|
|
return FakeResp_openai_gateway(
|
|
status=400, payload=_rejection_openai_gateway(key)
|
|
)
|
|
return FakeResp_openai_gateway(
|
|
payload={
|
|
"id": "x",
|
|
"model": body.get("model"),
|
|
"choices": [{"message": {"content": "hi there"}}],
|
|
}
|
|
)
|
|
|
|
async def aclose(self):
|
|
pass
|
|
|
|
|
|
class FakeStubbornClient_openai_gateway(FakeStrictClient_openai_gateway):
|
|
async def send(self, request, stream=False):
|
|
self.calls.append((request.url, dict(request.json_body or {})))
|
|
return FakeResp_openai_gateway(
|
|
status=400, payload=_rejection_openai_gateway("thinking")
|
|
)
|
|
|
|
|
|
class FakeReasoningStrictClient_openai_gateway(FakeStrictClient_openai_gateway):
|
|
rejected = ("reasoning",)
|
|
|
|
|
|
def _shape_cfg_openai_gateway():
|
|
svc = GatewayService()
|
|
cfg = svc.effective_config()
|
|
cfg["gateway_vision_enabled"] = False
|
|
cfg["gateway_max_retries"] = 0
|
|
return svc, cfg
|
|
|
|
|
|
def _chat_openai_gateway(rt, cfg, owner, stream=False):
|
|
body = {"messages": [{"role": "user", "content": "hi"}]}
|
|
if stream:
|
|
body["stream"] = True
|
|
return run_async(rt.handle_chat(body, cfg, ("guest", owner), "test", "default"))
|
|
|
|
|
|
def test_chat_auto_resolves_a_rejected_thinking_key_and_remembers_it(local_db, monkeypatch):
|
|
from devplacepy.services.openai_gateway.learned import learned_store
|
|
|
|
monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeStrictClient_openai_gateway)
|
|
svc, cfg = _shape_cfg_openai_gateway()
|
|
model = cfg["gateway_model"]
|
|
learned_store.forget("default", model)
|
|
try:
|
|
rt = svc.runtime()
|
|
response = _chat_openai_gateway(rt, cfg, "shape_resolve_first")
|
|
assert response.status_code == 200
|
|
assert len(rt._client.calls) == 2
|
|
assert "thinking" in rt._client.calls[0][1]
|
|
assert "thinking" not in rt._client.calls[1][1]
|
|
learned = learned_store.get("default", model)
|
|
assert learned is not None and learned.thinking_dialect == "none"
|
|
row = get_table("gateway_usage_ledger").find_one(owner_id="shape_resolve_first")
|
|
assert row["resolved_method"] == "none"
|
|
assert row["success"] == 1
|
|
second = _chat_openai_gateway(rt, cfg, "shape_resolve_second")
|
|
assert second.status_code == 200
|
|
assert len(rt._client.calls) == 3
|
|
assert "thinking" not in rt._client.calls[2][1]
|
|
finally:
|
|
learned_store.forget("default", model)
|
|
|
|
|
|
def test_chat_auto_resolve_works_for_streams(local_db, monkeypatch):
|
|
from devplacepy.services.openai_gateway.learned import learned_store
|
|
|
|
monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeStrictClient_openai_gateway)
|
|
svc, cfg = _shape_cfg_openai_gateway()
|
|
model = cfg["gateway_model"]
|
|
learned_store.forget("default", model)
|
|
try:
|
|
rt = svc.runtime()
|
|
response = _chat_openai_gateway(rt, cfg, "shape_resolve_stream", stream=True)
|
|
|
|
async def drain():
|
|
out = []
|
|
async for chunk in response.body_iterator:
|
|
out.append(chunk if isinstance(chunk, str) else chunk.decode())
|
|
return "".join(out)
|
|
|
|
assert "[DONE]" in run_async(drain())
|
|
assert len(rt._client.calls) == 2
|
|
assert learned_store.get("default", model) is not None
|
|
finally:
|
|
learned_store.forget("default", model)
|
|
|
|
|
|
def test_chat_genuine_bad_request_is_not_auto_resolved(local_db, monkeypatch):
|
|
from devplacepy.services.openai_gateway.learned import learned_store
|
|
|
|
monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeBadRequestClient_openai_gateway)
|
|
svc, cfg = _shape_cfg_openai_gateway()
|
|
model = cfg["gateway_model"]
|
|
learned_store.forget("default", model)
|
|
rt = svc.runtime()
|
|
response = _chat_openai_gateway(rt, cfg, "shape_genuine_bad")
|
|
assert response.status_code == 400
|
|
assert len(rt._client.calls) == 1
|
|
assert learned_store.get("default", model) is None
|
|
|
|
|
|
def test_chat_exhausted_resolution_returns_the_original_error_and_stores_nothing(
|
|
local_db, monkeypatch
|
|
):
|
|
from devplacepy.services.openai_gateway.learned import learned_store
|
|
from devplacepy.services.openai_gateway.shape import MAX_RESOLVE_ATTEMPTS
|
|
|
|
monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeStubbornClient_openai_gateway)
|
|
svc, cfg = _shape_cfg_openai_gateway()
|
|
model = cfg["gateway_model"]
|
|
learned_store.forget("default", model)
|
|
rt = svc.runtime()
|
|
response = _chat_openai_gateway(rt, cfg, "shape_exhausted")
|
|
assert response.status_code == 400
|
|
assert "thinking" in response.body.decode()
|
|
assert len(rt._client.calls) <= 1 + MAX_RESOLVE_ATTEMPTS
|
|
assert learned_store.get("default", model) is None
|
|
|
|
|
|
def test_chat_rejected_learned_method_is_forgotten_and_ladder_restarts(local_db, monkeypatch):
|
|
from devplacepy.services.openai_gateway.learned import learned_store
|
|
from devplacepy.services.openai_gateway.shape import ShapeMethod
|
|
|
|
monkeypatch.setattr(
|
|
gwmod.httpx, "AsyncClient", FakeReasoningStrictClient_openai_gateway
|
|
)
|
|
svc, cfg = _shape_cfg_openai_gateway()
|
|
model = cfg["gateway_model"]
|
|
learned_store.remember("default", model, ShapeMethod("openrouter"))
|
|
try:
|
|
rt = svc.runtime()
|
|
response = _chat_openai_gateway(rt, cfg, "shape_heal")
|
|
assert response.status_code == 200
|
|
assert "reasoning" in rt._client.calls[0][1]
|
|
assert "thinking" in rt._client.calls[1][1]
|
|
assert learned_store.get("default", model) is None
|
|
finally:
|
|
learned_store.forget("default", model)
|
|
|
|
|
|
def test_fallback_attempt_is_shaped_with_its_own_learned_method(local_db, monkeypatch):
|
|
from devplacepy.services.openai_gateway import routing
|
|
from devplacepy.services.openai_gateway.learned import learned_store
|
|
from devplacepy.services.openai_gateway.shape import ShapeMethod
|
|
|
|
class FakeFallbackShapeClient(FakeStrictClient_openai_gateway):
|
|
rejected = ()
|
|
|
|
async def send(self, request, stream=False):
|
|
body = request.json_body or {}
|
|
self.calls.append((request.url, dict(body)))
|
|
if body.get("model") == "shape-primary-target":
|
|
return FakeResp_openai_gateway(status=503, payload={"error": {"message": "down"}})
|
|
return FakeResp_openai_gateway(
|
|
payload={
|
|
"id": "x",
|
|
"model": body.get("model"),
|
|
"choices": [{"message": {"content": "ok"}}],
|
|
}
|
|
)
|
|
|
|
monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeFallbackShapeClient)
|
|
routing.model_store.set(
|
|
routing.ModelRouteIn(source_model="shape-backup", target_model="shape-backup-target")
|
|
)
|
|
routing.model_store.set(
|
|
routing.ModelRouteIn(
|
|
source_model="shape-primary",
|
|
target_model="shape-primary-target",
|
|
fallback_model="shape-backup",
|
|
)
|
|
)
|
|
learned_store.remember("default", "shape-backup-target", ShapeMethod("none"))
|
|
try:
|
|
svc, cfg = _shape_cfg_openai_gateway()
|
|
rt = svc.runtime()
|
|
response = run_async(
|
|
rt.handle_chat(
|
|
{"model": "shape-primary", "messages": [{"role": "user", "content": "hi"}]},
|
|
cfg,
|
|
("guest", "shape_fallback"),
|
|
"test",
|
|
"default",
|
|
)
|
|
)
|
|
assert response.status_code == 200
|
|
assert "thinking" in rt._client.calls[0][1]
|
|
assert rt._client.calls[-1][1]["model"] == "shape-backup-target"
|
|
assert "thinking" not in rt._client.calls[-1][1]
|
|
finally:
|
|
learned_store.forget("default", "shape-backup-target")
|
|
routing.model_store.remove("shape-primary")
|
|
routing.model_store.remove("shape-backup")
|
|
|
|
|
|
def _target_send_openai_gateway(rt, cfg, client, url, model):
|
|
import asyncio
|
|
|
|
async def go():
|
|
return await rt._send(
|
|
client,
|
|
asyncio.Semaphore(4),
|
|
"POST",
|
|
url,
|
|
{},
|
|
{**cfg, "gateway_max_retries": 0},
|
|
lambda message: None,
|
|
json_body={"model": model},
|
|
)
|
|
|
|
return run_async(go())
|
|
|
|
|
|
def _target_runtime_openai_gateway():
|
|
svc = GatewayService()
|
|
return svc.runtime(), svc.effective_config()
|
|
|
|
|
|
def test_trip_opens_below_threshold_and_ignores_threshold():
|
|
from devplacepy.services.openai_gateway.reliability import CircuitBreaker
|
|
|
|
breaker = CircuitBreaker(5, 30)
|
|
breaker.trip()
|
|
assert breaker.is_open is True
|
|
assert breaker.allow() is False
|
|
|
|
|
|
def test_a_failed_target_is_skipped_without_an_upstream_call(local_db):
|
|
rt, cfg = _target_runtime_openai_gateway()
|
|
client = FakeAlwaysFailClient_openai_gateway()
|
|
url = "https://target-a.example/v1/chat/completions"
|
|
first = _target_send_openai_gateway(rt, cfg, client, url, "m1")
|
|
assert first[0].status_code == 500
|
|
second = _target_send_openai_gateway(rt, cfg, client, url, "m1")
|
|
assert second[2]["circuit_open"] is True
|
|
assert len(client.calls) == 1
|
|
|
|
|
|
def test_one_bad_request_does_not_block_another_target(local_db):
|
|
rt, cfg = _target_runtime_openai_gateway()
|
|
client = FakeAlwaysFailClient_openai_gateway()
|
|
url = "https://target-b.example/v1/chat/completions"
|
|
_target_send_openai_gateway(rt, cfg, client, url, "model-b1")
|
|
other = _target_send_openai_gateway(rt, cfg, client, url, "model-b2")
|
|
assert other[2]["circuit_open"] is False
|
|
assert len(client.calls) == 2
|
|
|
|
|
|
def test_target_breaker_kill_switch_keeps_global_behaviour(local_db):
|
|
rt, cfg = _target_runtime_openai_gateway()
|
|
cfg = {**cfg, "gateway_target_circuit_enabled": False}
|
|
client = FakeAlwaysFailClient_openai_gateway()
|
|
url = "https://target-d.example/v1/chat/completions"
|
|
_target_send_openai_gateway(rt, cfg, client, url, "m1")
|
|
again = _target_send_openai_gateway(rt, cfg, client, url, "m1")
|
|
assert again[2]["circuit_open"] is False
|
|
assert len(client.calls) == 2
|
|
|
|
|
|
def test_shape_fixable_400_does_not_trip_the_target(local_db):
|
|
rt, cfg = _target_runtime_openai_gateway()
|
|
resp = FakeResp_openai_gateway(
|
|
status=400,
|
|
payload={"issues": [{"code": "unrecognized_keys", "keys": ["thinking"]}]},
|
|
)
|
|
assert gwmod._trip_worthy(resp) is False
|
|
plain = FakeResp_openai_gateway(status=400, payload={"error": "bad schema"})
|
|
assert gwmod._trip_worthy(plain) is True
|
|
|
|
|
|
def test_embedded_200_error_trips_the_target(local_db):
|
|
rt, cfg = _target_runtime_openai_gateway()
|
|
resp = FakeResp_openai_gateway(status=200, payload={"error": {"message": "provider exploded"}})
|
|
url = "https://target-e.example/v1/chat/completions"
|
|
assert rt._embedded_error(resp, False, url, "m1", cfg) is not None
|
|
assert rt._target_breaker("m1", cfg).is_open is True
|
|
assert rt.open_targets()[0]["model"] == "m1"
|
|
|
|
|
|
def test_transition_notifies_each_admin_once_with_detail_and_no_key(local_db, monkeypatch):
|
|
sent = []
|
|
monkeypatch.setattr(gwmod.background, "submit", lambda fn, *a, **k: sent.append((fn, a)))
|
|
rt, cfg = _target_runtime_openai_gateway()
|
|
url = "https://target-f.example/v1/chat/completions"
|
|
rt._trip_target("m1", url, cfg, "upstream_error", 500, "Bearer sk-secret123 failed", 1)
|
|
rt._trip_target("m1", url, cfg, "upstream_error", 500, "again", 1)
|
|
assert len(sent) == 1
|
|
detail = sent[0][1][1]
|
|
assert detail["category"] == "upstream_error"
|
|
assert detail["status"] == 500
|
|
assert "secret123" not in detail["upstream"]
|
|
assert detail["provider"] == "target-f.example"
|
|
|
|
|
|
def test_breaker_is_per_model_across_upstreams(local_db):
|
|
rt, cfg = _target_runtime_openai_gateway()
|
|
client = FakeAlwaysFailClient_openai_gateway()
|
|
_target_send_openai_gateway(rt, cfg, client, "https://target-g1.example/v1/chat/completions", "shared-model")
|
|
second = _target_send_openai_gateway(rt, cfg, client, "https://target-g2.example/v1/chat/completions", "shared-model")
|
|
assert second[2]["circuit_open"] is True
|
|
assert len(client.calls) == 1
|
|
|
|
|
|
def test_a_request_without_a_model_never_uses_the_target_breaker(local_db):
|
|
rt, cfg = _target_runtime_openai_gateway()
|
|
client = FakeAlwaysFailClient_openai_gateway()
|
|
url = "https://target-h.example/v1/other"
|
|
_target_send_openai_gateway(rt, cfg, client, url, "")
|
|
again = _target_send_openai_gateway(rt, cfg, client, url, "")
|
|
assert again[2]["circuit_open"] is False
|
|
assert rt._target_breakers == {}
|
|
|
|
|
|
class FakeNotifyClient_openai_gateway:
|
|
def __init__(self, *a, **k):
|
|
self.calls = []
|
|
|
|
def build_request(self, method, url, headers=None, json=None, content=None):
|
|
return FakeRequest(method, url, json)
|
|
|
|
async def send(self, request, stream=False):
|
|
self.calls.append((request.url, request.json_body))
|
|
body = request.json_body or {}
|
|
if body.get("model") == "nb-primary-target":
|
|
return FakeResp_openai_gateway(
|
|
status=500, payload={"error": "boom sk-secret-xyz"}
|
|
)
|
|
return FakeResp_openai_gateway(
|
|
payload={
|
|
"id": "x",
|
|
"model": body.get("model"),
|
|
"choices": [{"message": {"content": "notify ok"}}],
|
|
}
|
|
)
|
|
|
|
async def aclose(self):
|
|
pass
|
|
|
|
|
|
class FakeNotifyChainClient_openai_gateway:
|
|
def __init__(self, *a, **k):
|
|
self.calls = []
|
|
|
|
def build_request(self, method, url, headers=None, json=None, content=None):
|
|
return FakeRequest(method, url, json)
|
|
|
|
async def send(self, request, stream=False):
|
|
self.calls.append((request.url, request.json_body))
|
|
body = request.json_body or {}
|
|
if body.get("model") in ("n3-a-target", "n3-b-target"):
|
|
return FakeResp_openai_gateway(status=500, payload={"error": "boom"})
|
|
return FakeResp_openai_gateway(
|
|
payload={
|
|
"id": "x",
|
|
"model": body.get("model"),
|
|
"choices": [{"message": {"content": "chain ok"}}],
|
|
}
|
|
)
|
|
|
|
async def aclose(self):
|
|
pass
|
|
|
|
|
|
def _notify_chat_openai_gateway(monkeypatch, client_cls, routes, source, owner_id):
|
|
from devplacepy.services.openai_gateway import routing
|
|
|
|
monkeypatch.setattr(gwmod.httpx, "AsyncClient", client_cls)
|
|
for payload in routes:
|
|
routing.model_store.set(routing.ModelRouteIn(**payload))
|
|
sent = []
|
|
def _capture_notify_openai_gateway(fn, *a, **k):
|
|
if fn is gwmod._notify_gateway_status:
|
|
sent.append((fn, a))
|
|
monkeypatch.setattr(gwmod.background, "submit", _capture_notify_openai_gateway)
|
|
try:
|
|
svc = GatewayService()
|
|
cfg = svc.effective_config()
|
|
cfg["gateway_vision_enabled"] = False
|
|
cfg["gateway_max_retries"] = 0
|
|
rt = svc.runtime()
|
|
response = run_async(
|
|
rt.handle_chat(
|
|
{"model": source, "messages": [{"role": "user", "content": "hi"}]},
|
|
cfg,
|
|
("guest", owner_id),
|
|
"test",
|
|
"default",
|
|
)
|
|
)
|
|
return response, sent
|
|
finally:
|
|
for payload in routes:
|
|
routing.model_store.remove(payload["source_model"])
|
|
|
|
|
|
def _rendered_messages_openai_gateway(monkeypatch, details):
|
|
messages = []
|
|
|
|
class FakeUsersTable_openai_gateway:
|
|
def find(self, **kwargs):
|
|
if kwargs.get("role") == "Admin":
|
|
return [{"uid": "admin-1"}]
|
|
return []
|
|
|
|
monkeypatch.setattr(
|
|
"devplacepy.database.get_table",
|
|
lambda name: FakeUsersTable_openai_gateway(),
|
|
)
|
|
monkeypatch.setattr(
|
|
"devplacepy.utils.create_notification",
|
|
lambda uid, kind, message, *a, **k: messages.append(message),
|
|
)
|
|
for detail in details:
|
|
gwmod._notify_gateway_status(True, detail)
|
|
return messages
|
|
|
|
|
|
def test_trip_notification_carries_local_config_redirect_and_outcome(local_db, monkeypatch):
|
|
routes = [
|
|
{"source_model": "nb-backup", "target_model": "nb-backup-target"},
|
|
{
|
|
"source_model": "nb-primary",
|
|
"target_model": "nb-primary-target",
|
|
"fallback_model": "nb-backup",
|
|
},
|
|
]
|
|
response, sent = _notify_chat_openai_gateway(
|
|
monkeypatch, FakeNotifyClient_openai_gateway, routes, "nb-primary", "notify_one"
|
|
)
|
|
assert response.status_code == 200
|
|
assert len(sent) == 1
|
|
assert sent[0][0] is gwmod._notify_gateway_status
|
|
detail = sent[0][1][1]
|
|
assert detail["route"] == "nb-primary"
|
|
assert detail["model"] == "nb-primary-target"
|
|
assert detail["requested"] == "nb-primary"
|
|
assert "nb-primary" in detail["chain"]
|
|
assert "nb-backup" in detail["chain"]
|
|
assert detail["redirect"] == "nb-backup"
|
|
assert detail["outcome_ok"] is True
|
|
assert detail["served_route"] == "nb-backup"
|
|
assert "nb-backup" in detail["outcome"]
|
|
messages = _rendered_messages_openai_gateway(monkeypatch, [detail])
|
|
assert len(messages) == 1
|
|
assert "tripped its circuit breaker" in messages[0]
|
|
assert "Local config: 'nb-primary'" in messages[0]
|
|
assert "Full route:" in messages[0]
|
|
assert "nb-primary" in messages[0].split("Full route:")[1]
|
|
assert "nb-backup" in messages[0].split("Full route:")[1]
|
|
assert "Redirect: will redirect to fallback 'nb-backup'" in messages[0]
|
|
assert "Outcome: fallback 'nb-backup' answered HTTP 200 ok" in messages[0]
|
|
assert "Error class: upstream_error" in messages[0]
|
|
assert "HTTP status: 500" in messages[0]
|
|
assert "sk-secret-xyz" not in messages[0]
|
|
assert "[redacted]" in messages[0]
|
|
|
|
|
|
def test_trip_notifications_cover_every_hop_of_a_recursive_chain(local_db, monkeypatch):
|
|
routes = [
|
|
{"source_model": "n3-c", "target_model": "n3-c-target"},
|
|
{
|
|
"source_model": "n3-b",
|
|
"target_model": "n3-b-target",
|
|
"fallback_model": "n3-c",
|
|
},
|
|
{
|
|
"source_model": "n3-a",
|
|
"target_model": "n3-a-target",
|
|
"fallback_model": "n3-b",
|
|
},
|
|
]
|
|
response, sent = _notify_chat_openai_gateway(
|
|
monkeypatch, FakeNotifyChainClient_openai_gateway, routes, "n3-a", "notify_chain"
|
|
)
|
|
assert response.status_code == 200
|
|
assert len(sent) == 2
|
|
first = sent[0][1][1]
|
|
second = sent[1][1][1]
|
|
assert first["route"] == "n3-a"
|
|
assert first["redirect"] == "n3-b"
|
|
assert second["route"] == "n3-b"
|
|
assert second["redirect"] == "n3-c"
|
|
for detail in (first, second):
|
|
assert detail["requested"] == "n3-a"
|
|
assert "n3-a" in detail["chain"]
|
|
assert "n3-b" in detail["chain"]
|
|
assert "n3-c" in detail["chain"]
|
|
assert detail["outcome_ok"] is True
|
|
assert detail["outcome"] == "fallback 'n3-c' answered HTTP 200 ok"
|
|
assert detail["served_route"] == "n3-c"
|
|
messages = _rendered_messages_openai_gateway(monkeypatch, [first, second])
|
|
assert "Redirect: will redirect to fallback 'n3-b'" in messages[0]
|
|
assert "Redirect: will redirect to fallback 'n3-c'" in messages[1]
|
|
assert "Outcome: fallback 'n3-c' answered HTTP 200 ok" in messages[0]
|
|
assert "Outcome: fallback 'n3-c' answered HTTP 200 ok" in messages[1]
|
|
|
|
|
|
def test_trip_notification_reports_failed_outcome_when_every_fallback_fails(
|
|
local_db, monkeypatch
|
|
):
|
|
routes = [
|
|
{"source_model": "n4-b", "target_model": "n4-b-target"},
|
|
{
|
|
"source_model": "n4-a",
|
|
"target_model": "n4-a-target",
|
|
"fallback_model": "n4-b",
|
|
},
|
|
]
|
|
response, sent = _notify_chat_openai_gateway(
|
|
monkeypatch, FakeAlwaysFailClient_openai_gateway, routes, "n4-a", "notify_fail"
|
|
)
|
|
assert response.status_code == 500
|
|
assert len(sent) == 2
|
|
first = sent[0][1][1]
|
|
second = sent[1][1][1]
|
|
assert first["route"] == "n4-a"
|
|
assert first["redirect"] == "n4-b"
|
|
assert second["route"] == "n4-b"
|
|
assert second["redirect"] == ""
|
|
for detail in (first, second):
|
|
assert detail["outcome_ok"] is False
|
|
assert "all 2 route(s) failed" in detail["outcome"]
|
|
assert "HTTP 500" in detail["outcome"]
|
|
messages = _rendered_messages_openai_gateway(monkeypatch, [first, second])
|
|
assert "Redirect: will redirect to fallback 'n4-b'" in messages[0]
|
|
assert "Redirect: none, end of the fallback line" in messages[1]
|
|
assert "Outcome: all 2 route(s) failed" in messages[1]
|
|
|
|
|
|
def test_trip_notification_redirects_unrouted_default_to_first_fallback(
|
|
local_db, monkeypatch
|
|
):
|
|
rt, cfg = _target_runtime_openai_gateway()
|
|
sent = []
|
|
monkeypatch.setattr(
|
|
gwmod.background, "submit", lambda fn, *a, **k: sent.append((fn, a))
|
|
)
|
|
detail = gwmod._trip_detail(
|
|
"default-target",
|
|
"https://default.example/v1/chat/completions",
|
|
30,
|
|
"upstream_error",
|
|
500,
|
|
"boom",
|
|
0,
|
|
route="",
|
|
)
|
|
rt._notify_trips(
|
|
[{"kind": "target", "detail": detail}],
|
|
"chat",
|
|
"molodetz",
|
|
"weird-model",
|
|
[{"route": ""}, {"route": "m-pro"}],
|
|
True,
|
|
"fallback 'm-pro' answered HTTP 200 ok",
|
|
"m-pro",
|
|
)
|
|
assert len(sent) == 1
|
|
enriched = sent[0][1][1]
|
|
assert enriched["redirect"] == "m-pro"
|
|
messages = _rendered_messages_openai_gateway(monkeypatch, [enriched])
|
|
assert "Local config: default (no routed model)" in messages[0]
|
|
assert "Redirect: will redirect to fallback 'm-pro'" in messages[0]
|
|
assert "Outcome: fallback 'm-pro' answered HTTP 200 ok" in messages[0]
|
|
|
|
|
|
def test_vision_trip_notifies_with_local_config_and_no_fallback(local_db, monkeypatch):
|
|
import httpx
|
|
|
|
from devplacepy.services.openai_gateway.reliability import CircuitBreaker
|
|
from devplacepy.services.openai_gateway.vision import VisionAugmenter, VisionCache
|
|
|
|
class FailVisionClient_openai_gateway:
|
|
async def post(self, url, headers=None, json=None):
|
|
raise httpx.ConnectError("vision down")
|
|
|
|
breaker = CircuitBreaker(1, 60)
|
|
sink = []
|
|
augmenter = VisionAugmenter(
|
|
"https://vision.example/v1/chat/completions",
|
|
"vis-model",
|
|
"k",
|
|
VisionCache(0),
|
|
breaker=breaker,
|
|
max_retries=0,
|
|
trip_sink=sink,
|
|
)
|
|
image = {"type": "image_url", "image_url": {"url": "http://x/y.png"}}
|
|
first = run_async(augmenter._describe_one(FailVisionClient_openai_gateway(), image))
|
|
assert first.startswith("[vision call failed")
|
|
assert breaker.is_open is True
|
|
assert len(sink) == 1
|
|
assert sink[0]["kind"] == "vision"
|
|
assert sink[0]["detail"]["route"] == "vision:vis-model"
|
|
second = run_async(augmenter._describe_one(FailVisionClient_openai_gateway(), image))
|
|
assert second.startswith("[vision unavailable")
|
|
assert len(sink) == 1
|
|
rt, cfg = _target_runtime_openai_gateway()
|
|
sent = []
|
|
monkeypatch.setattr(
|
|
gwmod.background, "submit", lambda fn, *a, **k: sent.append((fn, a))
|
|
)
|
|
rt._notify_trips(
|
|
sink,
|
|
"chat",
|
|
"molodetz",
|
|
"molodetz",
|
|
[{"route": "molodetz"}],
|
|
True,
|
|
"fallback 'molodetz' answered HTTP 200 ok",
|
|
"molodetz",
|
|
)
|
|
assert len(sent) == 1
|
|
messages = _rendered_messages_openai_gateway(monkeypatch, [sent[0][1][1]])
|
|
assert "Local config: 'vision:vis-model'" in messages[0]
|
|
assert "Redirect: none, vision has no fallback model" in messages[0]
|
|
assert "Outcome: fallback 'molodetz' answered HTTP 200 ok" in messages[0]
|
|
|
|
|
|
def test_trip_notification_outcome_scrubs_secrets(local_db, monkeypatch):
|
|
detail = gwmod._trip_detail(
|
|
"s1-target",
|
|
"https://s1.example/v1/chat/completions",
|
|
30,
|
|
"connection",
|
|
"no response",
|
|
"boom",
|
|
0,
|
|
route="s1",
|
|
)
|
|
detail["requested"] = "s1"
|
|
detail["chain"] = "s1 (s1-target @ default)"
|
|
detail["redirect"] = ""
|
|
secret = "sk" + "-abc123"
|
|
detail["outcome"] = "connection failed (Bearer " + secret + ")"
|
|
messages = _rendered_messages_openai_gateway(monkeypatch, [detail])
|
|
assert secret not in messages[0]
|
|
assert "[redacted]" in messages[0]
|
|
|
|
|
|
def test_passthrough_global_trip_carries_local_config(local_db, monkeypatch):
|
|
monkeypatch.setattr(
|
|
gwmod.httpx, "AsyncClient", FakeAlwaysFailClient_openai_gateway
|
|
)
|
|
svc = GatewayService()
|
|
cfg = svc.effective_config()
|
|
cfg["gateway_circuit_threshold"] = 1
|
|
cfg["gateway_max_retries"] = 0
|
|
rt = svc.runtime()
|
|
sent = []
|
|
monkeypatch.setattr(
|
|
gwmod.background,
|
|
"submit",
|
|
lambda fn, *a, **k: sent.append((fn, a))
|
|
if fn is gwmod._notify_gateway_status
|
|
else None,
|
|
)
|
|
resp = run_async(
|
|
rt.handle_passthrough(
|
|
"POST",
|
|
"responses",
|
|
"application/json",
|
|
b"{}",
|
|
cfg,
|
|
("guest", "ptrip_local"),
|
|
"test",
|
|
"default",
|
|
)
|
|
)
|
|
assert resp.status_code == 500
|
|
assert len(sent) == 1
|
|
detail = sent[0][1][1]
|
|
assert detail.get("global") is True
|
|
assert detail["route"] == cfg["gateway_model"]
|
|
assert "no fallback line" in detail["chain"]
|
|
messages = _rendered_messages_openai_gateway(monkeypatch, [detail])
|
|
assert f"Local config: '{cfg['gateway_model']}'" in messages[0]
|
|
assert "passthrough POST responses (no fallback line)" in messages[0]
|
|
assert "Outcome: all 1 route(s) failed" in messages[0]
|