forked from retoor/devplacepy
Comment and post layouts stay full width on small screens. Secondary actions live in the More menu. Notifications can be grouped and throttled, and Enter can submit when that profile setting is on. Emulated-tool models accept large catalogs, with reserved names aliased on the wire and the catalog cached per owner. A dropped service-worker navigation is retried once. The test server waits 120s to finish importing before it is stopped.
3444 lines
116 KiB
Python
3444 lines
116 KiB
Python
# retoor <retoor@molodetz.nl>
|
|
|
|
import json
|
|
from starlette.requests import Request
|
|
from tests.conftest import BASE_URL, run_async
|
|
from devplacepy.database import get_table, set_setting
|
|
from devplacepy.utils import generate_uid
|
|
import devplacepy.services.openai_gateway.gateway as gwmod
|
|
from devplacepy.services.openai_gateway import GatewayService
|
|
from devplacepy.services.openai_gateway import auth_throttle
|
|
class FakeResp_openai_gateway:
|
|
def __init__(
|
|
self,
|
|
status=200,
|
|
payload=None,
|
|
ctype="application/json",
|
|
content=b"",
|
|
extra_headers=None,
|
|
):
|
|
self.status_code = status
|
|
self._payload = payload
|
|
self.text = json.dumps(payload) if payload is not None else ""
|
|
self.headers = {"content-type": ctype, **(extra_headers or {})}
|
|
self.content = content
|
|
|
|
def json(self):
|
|
if self._payload is None:
|
|
raise ValueError("no json")
|
|
return self._payload
|
|
|
|
async def aread(self):
|
|
return self.content or self.text.encode()
|
|
|
|
async def aclose(self):
|
|
pass
|
|
|
|
async def aiter_lines(self):
|
|
payload = self._payload or {}
|
|
chunk_id = payload.get("id", "x")
|
|
model = payload.get("model", "")
|
|
try:
|
|
content = payload["choices"][0]["message"].get("content") or ""
|
|
except (KeyError, IndexError, TypeError):
|
|
content = ""
|
|
try:
|
|
reasoning = payload["choices"][0]["message"].get("reasoning") or ""
|
|
except (KeyError, IndexError, TypeError):
|
|
reasoning = ""
|
|
|
|
def frame(delta=None, finish=None, usage=None):
|
|
body = {"id": chunk_id, "object": "chat.completion.chunk", "model": model}
|
|
if usage is not None:
|
|
body["choices"] = []
|
|
body["usage"] = usage
|
|
else:
|
|
body["choices"] = [{"index": 0, "delta": delta or {}, "finish_reason": finish}]
|
|
return f"data: {json.dumps(body)}"
|
|
|
|
yield frame({"role": "assistant"})
|
|
for i in range(0, len(reasoning), 5):
|
|
yield frame({"reasoning": reasoning[i : i + 5]})
|
|
for i in range(0, len(content), 5):
|
|
yield frame({"content": content[i : i + 5]})
|
|
yield frame({}, finish="stop")
|
|
if payload.get("usage"):
|
|
yield frame(usage=payload["usage"])
|
|
yield "data: [DONE]"
|
|
class FakeRequest:
|
|
def __init__(self, method, url, json_body, headers=None):
|
|
self.method = method
|
|
self.url = url
|
|
self.json_body = json_body
|
|
self.headers = headers or {}
|
|
self.extensions = {}
|
|
class FakeClient_openai_gateway:
|
|
def __init__(self, *a, **k):
|
|
self.calls = []
|
|
|
|
def build_request(self, method, url, headers=None, json=None, content=None):
|
|
return FakeRequest(method, url, json)
|
|
|
|
async def send(self, request, stream=False):
|
|
self.calls.append((request.url, request.json_body))
|
|
body = request.json_body or {}
|
|
return FakeResp_openai_gateway(
|
|
payload={
|
|
"id": "x",
|
|
"model": body.get("model"),
|
|
"choices": [{"message": {"content": "hi there"}}],
|
|
}
|
|
)
|
|
|
|
async def post(self, url, headers=None, json=None, timeout=None):
|
|
self.calls.append((url, json))
|
|
return FakeResp_openai_gateway(
|
|
payload={
|
|
"id": "x",
|
|
"model": json.get("model"),
|
|
"choices": [{"message": {"content": "hi there"}}],
|
|
}
|
|
)
|
|
|
|
async def request(self, method, url, headers=None, content=None):
|
|
return FakeResp_openai_gateway(payload={"ok": True})
|
|
|
|
async def aclose(self):
|
|
pass
|
|
def _make_request_openai_gateway(headers=None, cookies=None):
|
|
headers = headers or {}
|
|
raw = [(k.lower().encode(), v.encode()) for k, v in headers.items()]
|
|
if cookies:
|
|
raw.append(
|
|
(b"cookie", "; ".join(f"{k}={v}" for k, v in cookies.items()).encode())
|
|
)
|
|
return Request(
|
|
{
|
|
"type": "http",
|
|
"method": "POST",
|
|
"path": "/openai/v1/chat/completions",
|
|
"query_string": b"",
|
|
"headers": raw,
|
|
"state": {},
|
|
}
|
|
)
|
|
def _make_admin_openai_gateway(role="Admin"):
|
|
username = f"gw_{generate_uid()[:8]}"
|
|
api_key = generate_uid()
|
|
get_table("users").insert(
|
|
{
|
|
"uid": generate_uid(),
|
|
"username": username,
|
|
"terms_version": "1",
|
|
"email": f"{username}@t.dev",
|
|
"api_key": api_key,
|
|
"role": role,
|
|
"is_active": True,
|
|
}
|
|
)
|
|
return username, api_key
|
|
def _config(page, **fields):
|
|
page.request.post(f"{BASE_URL}/admin/services/openai/config", form=fields)
|
|
|
|
|
|
def test_model_is_forced(local_db, monkeypatch):
|
|
monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeClient_openai_gateway)
|
|
svc = GatewayService()
|
|
cfg = svc.effective_config()
|
|
cfg["gateway_force_model"] = True
|
|
cfg["gateway_model"] = "deepseek-chat"
|
|
rt = svc.runtime()
|
|
run_async(
|
|
rt.handle_chat(
|
|
{"model": "gpt-4", "messages": [{"role": "user", "content": "hi"}]},
|
|
cfg,
|
|
("guest", "test"),
|
|
"test",
|
|
"default",
|
|
)
|
|
)
|
|
assert rt._client.calls[-1][1]["model"] == "deepseek-chat"
|
|
|
|
|
|
def test_model_route_overrides_upstream(local_db, monkeypatch):
|
|
from devplacepy.services.openai_gateway import routing
|
|
|
|
monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeClient_openai_gateway)
|
|
routing.provider_store.set(
|
|
routing.ProviderIn(
|
|
name="gwroute",
|
|
base_url="https://routed.example/v1/chat/completions",
|
|
api_key="routed-key",
|
|
)
|
|
)
|
|
routing.model_store.set(
|
|
routing.ModelRouteIn(
|
|
source_model="custom-pro",
|
|
provider="gwroute",
|
|
target_model="vendor/pro",
|
|
kind="chat",
|
|
price_output_per_m=9.0,
|
|
)
|
|
)
|
|
try:
|
|
svc = GatewayService()
|
|
cfg = svc.effective_config()
|
|
cfg["gateway_vision_enabled"] = False
|
|
rt = svc.runtime()
|
|
run_async(
|
|
rt.handle_chat(
|
|
{"model": "custom-pro", "messages": [{"role": "user", "content": "hi"}]},
|
|
cfg,
|
|
("guest", "test"),
|
|
"test",
|
|
"default",
|
|
)
|
|
)
|
|
url, body = rt._client.calls[-1]
|
|
assert str(url) == "https://routed.example/v1/chat/completions"
|
|
assert body["model"] == "vendor/pro"
|
|
finally:
|
|
routing.model_store.remove("custom-pro")
|
|
routing.provider_store.remove("gwroute")
|
|
|
|
|
|
def test_vision_rewrites_image_to_text(local_db, monkeypatch):
|
|
monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeClient_openai_gateway)
|
|
svc = GatewayService()
|
|
cfg = svc.effective_config()
|
|
cfg["gateway_vision_enabled"] = True
|
|
cfg["gateway_vision_key"] = "" # no key -> placeholder text, still rewrites to str
|
|
rt = svc.runtime()
|
|
msgs = [
|
|
{
|
|
"role": "user",
|
|
"content": [
|
|
{"type": "text", "text": "what is this"},
|
|
{"type": "image_url", "image_url": {"url": "http://x/y.png"}},
|
|
],
|
|
}
|
|
]
|
|
run_async(rt.handle_chat({"messages": msgs}, cfg, ("guest", "test"), "test", "default"))
|
|
sent = rt._client.calls[-1][1]["messages"][0]["content"]
|
|
assert isinstance(sent, str) and "vision" in sent.lower()
|
|
|
|
|
|
def test_streaming_emits_sse(local_db, monkeypatch):
|
|
monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeClient_openai_gateway)
|
|
svc = GatewayService()
|
|
cfg = svc.effective_config()
|
|
rt = svc.runtime()
|
|
resp = run_async(
|
|
rt.handle_chat(
|
|
{"messages": [{"role": "user", "content": "hi"}], "stream": True},
|
|
cfg,
|
|
("guest", "test"),
|
|
"test",
|
|
"default",
|
|
)
|
|
)
|
|
|
|
async def drain():
|
|
out = []
|
|
async for chunk in resp.body_iterator:
|
|
out.append(chunk if isinstance(chunk, str) else chunk.decode())
|
|
return "".join(out)
|
|
|
|
body = run_async(drain())
|
|
assert "chat.completion.chunk" in body
|
|
assert "[DONE]" in body
|
|
|
|
|
|
def test_authorize_static_access_key(local_db):
|
|
set_setting("gateway_require_auth", "1")
|
|
set_setting("gateway_access_key", "topsecret")
|
|
try:
|
|
svc = GatewayService()
|
|
assert svc.authorize(_make_request_openai_gateway(headers={"X-API-KEY": "topsecret"})) is True
|
|
assert svc.authorize(_make_request_openai_gateway(headers={"X-API-KEY": "wrong"})) is False
|
|
finally:
|
|
set_setting("gateway_access_key", "")
|
|
set_setting("gateway_require_auth", "1")
|
|
auth_throttle.clear()
|
|
|
|
|
|
def test_authorize_admin_and_user_toggles(local_db):
|
|
set_setting("gateway_require_auth", "1")
|
|
set_setting("gateway_access_key", "")
|
|
set_setting("gateway_allow_admins", "1")
|
|
set_setting("gateway_allow_users", "0")
|
|
try:
|
|
_, admin_key = _make_admin_openai_gateway(role="Admin")
|
|
_, member_key = _make_admin_openai_gateway(role="Member")
|
|
svc = GatewayService()
|
|
|
|
assert (
|
|
svc.authorize(_make_request_openai_gateway(headers={"Authorization": f"Bearer {admin_key}"}))
|
|
is True
|
|
)
|
|
assert svc.authorize(_make_request_openai_gateway(headers={"X-API-KEY": member_key})) is False
|
|
|
|
set_setting("gateway_allow_users", "1")
|
|
assert svc.authorize(_make_request_openai_gateway(headers={"X-API-KEY": member_key})) is True
|
|
finally:
|
|
set_setting("gateway_allow_admins", "1")
|
|
set_setting("gateway_allow_users", "1")
|
|
set_setting("gateway_require_auth", "1")
|
|
set_setting("gateway_access_key", "")
|
|
auth_throttle.clear()
|
|
|
|
|
|
def test_authorize_require_auth_off_is_open(local_db):
|
|
set_setting("gateway_require_auth", "0")
|
|
try:
|
|
svc = GatewayService()
|
|
assert svc.authorize(_make_request_openai_gateway()) is True
|
|
finally:
|
|
set_setting("gateway_require_auth", "1")
|
|
|
|
|
|
def test_authorize_throttles_repeated_failed_auth_from_one_ip(local_db):
|
|
import pytest
|
|
from fastapi import HTTPException
|
|
|
|
set_setting("gateway_require_auth", "1")
|
|
set_setting("gateway_access_key", "")
|
|
set_setting("gateway_auth_throttle_max_failures", "3")
|
|
set_setting("gateway_auth_throttle_window_seconds", "60")
|
|
auth_throttle.clear()
|
|
try:
|
|
svc = GatewayService()
|
|
ip = "203.0.113.10"
|
|
for _ in range(3):
|
|
assert (
|
|
svc.authorize(_make_request_openai_gateway(headers={"X-Real-IP": ip}))
|
|
is False
|
|
)
|
|
with pytest.raises(HTTPException) as excinfo:
|
|
svc.authorize(_make_request_openai_gateway(headers={"X-Real-IP": ip}))
|
|
assert excinfo.value.status_code == 429
|
|
assert "Retry-After" in excinfo.value.headers
|
|
finally:
|
|
set_setting("gateway_auth_throttle_max_failures", "10")
|
|
set_setting("gateway_auth_throttle_window_seconds", "60")
|
|
set_setting("gateway_require_auth", "1")
|
|
auth_throttle.clear()
|
|
|
|
|
|
def test_authorize_throttle_does_not_affect_other_ips(local_db):
|
|
set_setting("gateway_require_auth", "1")
|
|
set_setting("gateway_access_key", "")
|
|
set_setting("gateway_auth_throttle_max_failures", "3")
|
|
set_setting("gateway_auth_throttle_window_seconds", "60")
|
|
auth_throttle.clear()
|
|
try:
|
|
svc = GatewayService()
|
|
throttled_ip = "203.0.113.20"
|
|
other_ip = "203.0.113.21"
|
|
for _ in range(3):
|
|
svc.authorize(_make_request_openai_gateway(headers={"X-Real-IP": throttled_ip}))
|
|
assert auth_throttle.is_throttled(throttled_ip, 3, 60) is True
|
|
assert (
|
|
svc.authorize(_make_request_openai_gateway(headers={"X-Real-IP": other_ip}))
|
|
is False
|
|
)
|
|
assert auth_throttle.is_throttled(other_ip, 3, 60) is False
|
|
finally:
|
|
set_setting("gateway_auth_throttle_max_failures", "10")
|
|
set_setting("gateway_auth_throttle_window_seconds", "60")
|
|
set_setting("gateway_require_auth", "1")
|
|
auth_throttle.clear()
|
|
|
|
|
|
def test_authorize_valid_credentials_succeed_from_throttled_ip(local_db):
|
|
set_setting("gateway_require_auth", "1")
|
|
set_setting("gateway_access_key", "")
|
|
set_setting("gateway_allow_users", "1")
|
|
set_setting("gateway_auth_throttle_max_failures", "3")
|
|
set_setting("gateway_auth_throttle_window_seconds", "60")
|
|
auth_throttle.clear()
|
|
try:
|
|
svc = GatewayService()
|
|
ip = "203.0.113.30"
|
|
_, member_key = _make_admin_openai_gateway(role="Member")
|
|
for _ in range(3):
|
|
svc.authorize(_make_request_openai_gateway(headers={"X-Real-IP": ip}))
|
|
assert auth_throttle.is_throttled(ip, 3, 60) is True
|
|
assert (
|
|
svc.authorize(
|
|
_make_request_openai_gateway(
|
|
headers={"X-Real-IP": ip, "X-API-KEY": member_key}
|
|
)
|
|
)
|
|
is True
|
|
)
|
|
set_setting("gateway_access_key", "topsecret")
|
|
assert (
|
|
svc.authorize(
|
|
_make_request_openai_gateway(
|
|
headers={"X-Real-IP": ip, "X-API-KEY": "topsecret"}
|
|
)
|
|
)
|
|
is True
|
|
)
|
|
finally:
|
|
set_setting("gateway_access_key", "")
|
|
set_setting("gateway_auth_throttle_max_failures", "10")
|
|
set_setting("gateway_auth_throttle_window_seconds", "60")
|
|
set_setting("gateway_require_auth", "1")
|
|
auth_throttle.clear()
|
|
|
|
|
|
def test_authorize_throttle_resets_after_window(local_db, monkeypatch):
|
|
set_setting("gateway_require_auth", "1")
|
|
set_setting("gateway_access_key", "")
|
|
set_setting("gateway_auth_throttle_max_failures", "3")
|
|
set_setting("gateway_auth_throttle_window_seconds", "60")
|
|
auth_throttle.clear()
|
|
try:
|
|
now = [2000.0]
|
|
monkeypatch.setattr(auth_throttle.time, "time", lambda: now[0])
|
|
svc = GatewayService()
|
|
ip = "203.0.113.40"
|
|
for _ in range(3):
|
|
svc.authorize(_make_request_openai_gateway(headers={"X-Real-IP": ip}))
|
|
assert auth_throttle.is_throttled(ip, 3, 60) is True
|
|
now[0] += 61
|
|
assert (
|
|
svc.authorize(_make_request_openai_gateway(headers={"X-Real-IP": ip}))
|
|
is False
|
|
)
|
|
finally:
|
|
set_setting("gateway_auth_throttle_max_failures", "10")
|
|
set_setting("gateway_auth_throttle_window_seconds", "60")
|
|
set_setting("gateway_require_auth", "1")
|
|
auth_throttle.clear()
|
|
|
|
|
|
class FakeEmbedClient_openai_gateway:
|
|
def __init__(self, *a, **k):
|
|
self.calls = []
|
|
|
|
def build_request(self, method, url, headers=None, json=None, content=None):
|
|
return FakeRequest(method, url, json)
|
|
|
|
async def send(self, request, stream=False):
|
|
self.calls.append((request.url, request.json_body))
|
|
body = request.json_body or {}
|
|
return FakeResp_openai_gateway(
|
|
payload={
|
|
"object": "list",
|
|
"model": body.get("model"),
|
|
"data": [{"object": "embedding", "index": 0, "embedding": [0.1, 0.2]}],
|
|
"usage": {"prompt_tokens": 5, "total_tokens": 5},
|
|
}
|
|
)
|
|
|
|
async def aclose(self):
|
|
pass
|
|
|
|
|
|
def test_embeddings_remaps_alias_to_configured_model(local_db, monkeypatch):
|
|
monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeEmbedClient_openai_gateway)
|
|
monkeypatch.setenv("OPENROUTER_API_KEY", "test-embed-key")
|
|
svc = GatewayService()
|
|
cfg = svc.effective_config()
|
|
cfg["gateway_force_model"] = False
|
|
cfg["gateway_embed_enabled"] = True
|
|
cfg["gateway_embed_model"] = "qwen/qwen3-embedding-8b"
|
|
rt = svc.runtime()
|
|
run_async(
|
|
rt.handle_embeddings(
|
|
{"model": "molodetz~embed", "input": "hello"},
|
|
cfg,
|
|
("guest", "test"),
|
|
"test",
|
|
"default",
|
|
)
|
|
)
|
|
assert rt._client.calls[-1][1]["model"] == "qwen/qwen3-embedding-8b"
|
|
|
|
|
|
def test_embeddings_success_records_ledger(local_db, monkeypatch):
|
|
monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeEmbedClient_openai_gateway)
|
|
monkeypatch.setenv("OPENROUTER_API_KEY", "test-embed-key")
|
|
svc = GatewayService()
|
|
cfg = svc.effective_config()
|
|
cfg["gateway_force_model"] = True
|
|
cfg["gateway_embed_enabled"] = True
|
|
cfg["gateway_embed_model"] = "qwen/qwen3-embedding-8b"
|
|
rt = svc.runtime()
|
|
before = rt.embed_calls
|
|
resp = run_async(
|
|
rt.handle_embeddings(
|
|
{"model": "molodetz~embed", "input": "hello"},
|
|
cfg,
|
|
("guest", "ledger_probe"),
|
|
"test",
|
|
"default",
|
|
)
|
|
)
|
|
assert resp.status_code == 200
|
|
payload = json.loads(bytes(resp.body).decode())
|
|
assert payload["data"][0]["embedding"] == [0.1, 0.2]
|
|
assert rt.embed_calls == before + 1
|
|
row = get_table("gateway_usage_ledger").find_one(owner_id="ledger_probe")
|
|
assert row is not None
|
|
assert row["backend"] == "embed"
|
|
assert row["endpoint"] == "embeddings"
|
|
assert row["requested_model"] == "molodetz~embed"
|
|
assert row["model"] == "qwen/qwen3-embedding-8b"
|
|
assert row["success"] == 1
|
|
|
|
|
|
def test_embeddings_disabled_returns_503(local_db, monkeypatch):
|
|
monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeEmbedClient_openai_gateway)
|
|
svc = GatewayService()
|
|
cfg = svc.effective_config()
|
|
cfg["gateway_embed_enabled"] = False
|
|
rt = svc.runtime()
|
|
resp = run_async(
|
|
rt.handle_embeddings(
|
|
{"input": "hello"}, cfg, ("guest", "test"), "test", "default"
|
|
)
|
|
)
|
|
assert resp.status_code == 503
|
|
|
|
|
|
def test_embeddings_missing_key_returns_clean_error_not_unauthenticated_call(
|
|
local_db, monkeypatch
|
|
):
|
|
monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeEmbedClient_openai_gateway)
|
|
monkeypatch.delenv("OPENROUTER_API_KEY", raising=False)
|
|
monkeypatch.delenv("DEEPSEEK_API_KEY", raising=False)
|
|
set_setting("gateway_api_key", "")
|
|
set_setting("gateway_vision_key", "")
|
|
set_setting("gateway_embed_key", "")
|
|
try:
|
|
svc = GatewayService()
|
|
cfg = svc.effective_config()
|
|
assert cfg["gateway_embed_key"] == ""
|
|
cfg["gateway_embed_enabled"] = True
|
|
rt = svc.runtime()
|
|
resp = run_async(
|
|
rt.handle_embeddings(
|
|
{"input": "hello"}, cfg, ("guest", "no_key_probe"), "test", "default"
|
|
)
|
|
)
|
|
assert resp.status_code == 503
|
|
payload = json.loads(bytes(resp.body).decode())
|
|
assert payload["error"]["type"] == "embeddings_not_configured"
|
|
assert rt._client is None
|
|
finally:
|
|
set_setting("gateway_api_key", "")
|
|
set_setting("gateway_vision_key", "")
|
|
set_setting("gateway_embed_key", "")
|
|
|
|
|
|
class FakeImageClient_openai_gateway:
|
|
def __init__(self, *a, **k):
|
|
self.calls = []
|
|
|
|
def build_request(self, method, url, headers=None, json=None, content=None):
|
|
return FakeRequest(method, url, json)
|
|
|
|
async def send(self, request, stream=False):
|
|
self.calls.append((request.url, request.json_body))
|
|
body = request.json_body or {}
|
|
return FakeResp_openai_gateway(
|
|
payload={
|
|
"created": 1,
|
|
"model": body.get("model"),
|
|
"data": [{"b64_json": "aGVsbG8="}],
|
|
"usage": {"cost": 0.05},
|
|
}
|
|
)
|
|
|
|
async def aclose(self):
|
|
pass
|
|
|
|
|
|
def test_images_remaps_alias_to_configured_model(local_db, monkeypatch):
|
|
monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeImageClient_openai_gateway)
|
|
svc = GatewayService()
|
|
cfg = svc.effective_config()
|
|
cfg["gateway_force_model"] = False
|
|
cfg["gateway_image_enabled"] = True
|
|
cfg["gateway_image_model"] = "black-forest-labs/flux.2-pro"
|
|
rt = svc.runtime()
|
|
run_async(
|
|
rt.handle_images(
|
|
{
|
|
"model": "molodetz-img-small",
|
|
"prompt": "award emblem",
|
|
"size": "512x512",
|
|
},
|
|
cfg,
|
|
("guest", "test"),
|
|
"test",
|
|
"default",
|
|
)
|
|
)
|
|
assert rt._client.calls[-1][1]["model"] == "black-forest-labs/flux.2-pro"
|
|
|
|
|
|
def test_image_route_overrides_upstream(local_db, monkeypatch):
|
|
from devplacepy.services.openai_gateway import routing
|
|
|
|
monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeImageClient_openai_gateway)
|
|
routing.provider_store.set(
|
|
routing.ProviderIn(
|
|
name="gwimg",
|
|
base_url="https://routed.example/v1/chat/completions",
|
|
api_key="img-key",
|
|
)
|
|
)
|
|
routing.model_store.set(
|
|
routing.ModelRouteIn(
|
|
source_model="custom-img",
|
|
provider="gwimg",
|
|
target_model="vendor/flux-pro",
|
|
kind="image",
|
|
price_input_per_m=0.06,
|
|
)
|
|
)
|
|
try:
|
|
svc = GatewayService()
|
|
cfg = svc.effective_config()
|
|
rt = svc.runtime()
|
|
run_async(
|
|
rt.handle_images(
|
|
{
|
|
"model": "custom-img",
|
|
"prompt": "trophy",
|
|
"response_format": "b64_json",
|
|
},
|
|
cfg,
|
|
("guest", "test"),
|
|
"test",
|
|
"default",
|
|
)
|
|
)
|
|
url, body = rt._client.calls[-1]
|
|
assert str(url) == "https://routed.example/v1/images"
|
|
assert body["model"] == "vendor/flux-pro"
|
|
finally:
|
|
routing.model_store.remove("custom-img")
|
|
routing.provider_store.remove("gwimg")
|
|
|
|
|
|
def test_images_success_records_ledger(local_db, monkeypatch):
|
|
monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeImageClient_openai_gateway)
|
|
svc = GatewayService()
|
|
cfg = svc.effective_config()
|
|
cfg["gateway_image_enabled"] = True
|
|
cfg["gateway_image_model"] = "black-forest-labs/flux.2-pro"
|
|
rt = svc.runtime()
|
|
before = rt.image_calls
|
|
resp = run_async(
|
|
rt.handle_images(
|
|
{"model": "molodetz-img-small", "prompt": "badge"},
|
|
cfg,
|
|
("guest", "img_ledger"),
|
|
"test",
|
|
"default",
|
|
)
|
|
)
|
|
assert resp.status_code == 200
|
|
payload = json.loads(bytes(resp.body).decode())
|
|
assert payload["data"][0]["b64_json"] == "aGVsbG8="
|
|
assert rt.image_calls == before + 1
|
|
row = get_table("gateway_usage_ledger").find_one(owner_id="img_ledger")
|
|
assert row is not None
|
|
assert row["backend"] == "image"
|
|
assert row["endpoint"] == "images/generations"
|
|
assert row["requested_model"] == "molodetz-img-small"
|
|
assert row["model"] == "black-forest-labs/flux.2-pro"
|
|
assert row["success"] == 1
|
|
assert float(row["cost_usd"]) == 0.05
|
|
|
|
|
|
def test_images_disabled_returns_503(local_db, monkeypatch):
|
|
monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeImageClient_openai_gateway)
|
|
svc = GatewayService()
|
|
cfg = svc.effective_config()
|
|
cfg["gateway_image_enabled"] = False
|
|
rt = svc.runtime()
|
|
resp = run_async(
|
|
rt.handle_images(
|
|
{"prompt": "badge"}, cfg, ("guest", "test"), "test", "default"
|
|
)
|
|
)
|
|
assert resp.status_code == 503
|
|
|
|
|
|
def test_compute_cost_embed_branch():
|
|
from devplacepy.services.openai_gateway.usage import (
|
|
Pricing,
|
|
compute_cost,
|
|
normalize_usage,
|
|
)
|
|
|
|
pricing = Pricing(
|
|
chat_cache_hit_per_m=0.0,
|
|
chat_cache_miss_per_m=0.0,
|
|
chat_output_per_m=0.0,
|
|
vision_input_per_m=0.0,
|
|
vision_output_per_m=0.0,
|
|
embed_input_per_m=0.01,
|
|
)
|
|
usage = {"prompt_tokens": 1_000_000, "total_tokens": 1_000_000}
|
|
norm = normalize_usage(usage)
|
|
total, input_cost, output_cost, native = compute_cost(
|
|
usage, norm, pricing, "embed"
|
|
)
|
|
assert native is False
|
|
assert output_cost == 0.0
|
|
assert abs(total - 0.01) < 1e-9
|
|
assert abs(input_cost - 0.01) < 1e-9
|
|
|
|
|
|
def test_app_reference_stored_in_ledger_for_chat(local_db, monkeypatch):
|
|
monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeClient_openai_gateway)
|
|
svc = GatewayService()
|
|
cfg = svc.effective_config()
|
|
cfg["gateway_force_model"] = True
|
|
cfg["gateway_model"] = "deepseek-chat"
|
|
rt = svc.runtime()
|
|
run_async(
|
|
rt.handle_chat(
|
|
{"messages": [{"role": "user", "content": "ref_test"}]},
|
|
cfg,
|
|
("guest", "ref_probe"),
|
|
"test-ua",
|
|
"my-custom-app",
|
|
)
|
|
)
|
|
row = get_table("gateway_usage_ledger").find_one(owner_id="ref_probe")
|
|
assert row is not None
|
|
assert row["app_reference"] == "my-custom-app"
|
|
assert row["backend"] == "chat"
|
|
|
|
|
|
def test_app_reference_defaults_when_not_passed(local_db, monkeypatch):
|
|
monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeClient_openai_gateway)
|
|
svc = GatewayService()
|
|
cfg = svc.effective_config()
|
|
cfg["gateway_force_model"] = True
|
|
cfg["gateway_model"] = "deepseek-chat"
|
|
rt = svc.runtime()
|
|
run_async(
|
|
rt.handle_chat(
|
|
{"messages": [{"role": "user", "content": "default_test"}]},
|
|
cfg,
|
|
("guest", "default_probe"),
|
|
"test-ua",
|
|
"default",
|
|
)
|
|
)
|
|
row = get_table("gateway_usage_ledger").find_one(owner_id="default_probe")
|
|
assert row is not None
|
|
assert row["app_reference"] == "default"
|
|
|
|
|
|
def test_app_reference_stored_in_ledger_for_embeddings(local_db, monkeypatch):
|
|
monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeEmbedClient_openai_gateway)
|
|
monkeypatch.setenv("OPENROUTER_API_KEY", "test-embed-key")
|
|
svc = GatewayService()
|
|
cfg = svc.effective_config()
|
|
cfg["gateway_embed_enabled"] = True
|
|
cfg["gateway_embed_model"] = "qwen/qwen3-embedding-8b"
|
|
rt = svc.runtime()
|
|
run_async(
|
|
rt.handle_embeddings(
|
|
{"input": "hello"},
|
|
cfg,
|
|
("guest", "embed_ref"),
|
|
"test-ua",
|
|
"devplace-embed-test-v-1-0-0",
|
|
)
|
|
)
|
|
row = get_table("gateway_usage_ledger").find_one(owner_id="embed_ref")
|
|
assert row is not None
|
|
assert row["app_reference"] == "devplace-embed-test-v-1-0-0"
|
|
assert row["backend"] == "embed"
|
|
|
|
|
|
def test_app_reference_stored_in_ledger_for_images(local_db, monkeypatch):
|
|
monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeImageClient_openai_gateway)
|
|
svc = GatewayService()
|
|
cfg = svc.effective_config()
|
|
cfg["gateway_image_enabled"] = True
|
|
cfg["gateway_image_model"] = "black-forest-labs/flux.2-pro"
|
|
rt = svc.runtime()
|
|
run_async(
|
|
rt.handle_images(
|
|
{"prompt": "ref badge"},
|
|
cfg,
|
|
("guest", "img_ref"),
|
|
"test-ua",
|
|
"devplace-image-test-v-1-0-0",
|
|
)
|
|
)
|
|
row = get_table("gateway_usage_ledger").find_one(owner_id="img_ref")
|
|
assert row is not None
|
|
assert row["app_reference"] == "devplace-image-test-v-1-0-0"
|
|
assert row["backend"] == "image"
|
|
|
|
|
|
def test_app_reference_column_exists(local_db):
|
|
db = get_table("gateway_usage_ledger").db
|
|
if "gateway_usage_ledger" not in db.tables:
|
|
return
|
|
rows = list(db.query("PRAGMA table_info('gateway_usage_ledger')"))
|
|
column_names = [r["name"] for r in rows]
|
|
assert "app_reference" in column_names
|
|
|
|
|
|
def test_chat_unknown_model_falls_back_to_default(local_db, monkeypatch):
|
|
monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeClient_openai_gateway)
|
|
svc = GatewayService()
|
|
cfg = svc.effective_config()
|
|
cfg["gateway_force_model"] = False
|
|
cfg["gateway_model"] = "deepseek-v4-flash"
|
|
rt = svc.runtime()
|
|
run_async(
|
|
rt.handle_chat(
|
|
{"model": "nonexistent-model-v99", "messages": [{"role": "user", "content": "hi"}]},
|
|
cfg,
|
|
("guest", "fallback_test_chat"),
|
|
"test",
|
|
"default",
|
|
)
|
|
)
|
|
assert rt._client.calls[-1][1]["model"] == "deepseek-v4-flash"
|
|
|
|
|
|
class FakeFallbackClient_openai_gateway:
|
|
def __init__(self, *a, **k):
|
|
self.calls = []
|
|
|
|
def build_request(self, method, url, headers=None, json=None, content=None):
|
|
return FakeRequest(method, url, json)
|
|
|
|
async def send(self, request, stream=False):
|
|
self.calls.append((request.url, request.json_body))
|
|
body = request.json_body or {}
|
|
model = body.get("model")
|
|
if model == "primary-target":
|
|
return FakeResp_openai_gateway(status=500, payload={"error": "boom"})
|
|
return FakeResp_openai_gateway(
|
|
payload={
|
|
"id": "x",
|
|
"model": model,
|
|
"choices": [{"message": {"content": "fallback ok"}}],
|
|
}
|
|
)
|
|
|
|
async def aclose(self):
|
|
pass
|
|
|
|
|
|
class FakeAlwaysFailClient_openai_gateway:
|
|
def __init__(self, *a, **k):
|
|
self.calls = []
|
|
|
|
def build_request(self, method, url, headers=None, json=None, content=None):
|
|
return FakeRequest(method, url, json)
|
|
|
|
async def send(self, request, stream=False):
|
|
self.calls.append((request.url, request.json_body))
|
|
return FakeResp_openai_gateway(status=500, payload={"error": "boom"})
|
|
|
|
async def aclose(self):
|
|
pass
|
|
|
|
|
|
class FakeBadRequestClient_openai_gateway:
|
|
def __init__(self, *a, **k):
|
|
self.calls = []
|
|
|
|
def build_request(self, method, url, headers=None, json=None, content=None):
|
|
return FakeRequest(method, url, json)
|
|
|
|
async def send(self, request, stream=False):
|
|
self.calls.append((request.url, request.json_body))
|
|
return FakeResp_openai_gateway(
|
|
status=400, payload={"error": {"message": "invalid request: bad param"}}
|
|
)
|
|
|
|
async def aclose(self):
|
|
pass
|
|
|
|
|
|
def test_chat_does_not_fall_back_on_an_unrecoverable_bad_request(local_db, monkeypatch):
|
|
from devplacepy.services.openai_gateway import routing
|
|
|
|
monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeBadRequestClient_openai_gateway)
|
|
routing.model_store.set(
|
|
routing.ModelRouteIn(source_model="fb-badreq-backup", target_model="backup-target")
|
|
)
|
|
routing.model_store.set(
|
|
routing.ModelRouteIn(
|
|
source_model="fb-badreq-primary",
|
|
target_model="primary-target",
|
|
fallback_model="fb-badreq-backup",
|
|
)
|
|
)
|
|
try:
|
|
svc = GatewayService()
|
|
cfg = svc.effective_config()
|
|
cfg["gateway_vision_enabled"] = False
|
|
cfg["gateway_max_retries"] = 0
|
|
rt = svc.runtime()
|
|
response = run_async(
|
|
rt.handle_chat(
|
|
{"model": "fb-badreq-primary", "messages": [{"role": "user", "content": "hi"}]},
|
|
cfg,
|
|
("guest", "no_fallback_on_bad_request"),
|
|
"test",
|
|
"default",
|
|
)
|
|
)
|
|
assert response.status_code == 400
|
|
assert len(rt._client.calls) == 1
|
|
finally:
|
|
routing.model_store.remove("fb-badreq-primary")
|
|
routing.model_store.remove("fb-badreq-backup")
|
|
|
|
|
|
def test_chat_falls_back_when_the_primary_model_fails(local_db, monkeypatch):
|
|
from devplacepy.services.openai_gateway import routing
|
|
|
|
monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeFallbackClient_openai_gateway)
|
|
routing.model_store.set(
|
|
routing.ModelRouteIn(source_model="fb-backup-route", target_model="backup-target")
|
|
)
|
|
routing.model_store.set(
|
|
routing.ModelRouteIn(
|
|
source_model="fb-primary-route",
|
|
target_model="primary-target",
|
|
fallback_model="fb-backup-route",
|
|
)
|
|
)
|
|
try:
|
|
svc = GatewayService()
|
|
cfg = svc.effective_config()
|
|
cfg["gateway_vision_enabled"] = False
|
|
cfg["gateway_max_retries"] = 0
|
|
rt = svc.runtime()
|
|
response = run_async(
|
|
rt.handle_chat(
|
|
{"model": "fb-primary-route", "messages": [{"role": "user", "content": "hi"}]},
|
|
cfg,
|
|
("guest", "fallback_chat_success"),
|
|
"test",
|
|
"default",
|
|
)
|
|
)
|
|
assert response.status_code == 200
|
|
assert rt._client.calls[0][1]["model"] == "primary-target"
|
|
assert rt._client.calls[-1][1]["model"] == "backup-target"
|
|
row = get_table("gateway_usage_ledger").find_one(owner_id="fallback_chat_success")
|
|
assert row is not None
|
|
assert row["requested_model"] == "fb-primary-route"
|
|
assert row["model"] == "backup-target"
|
|
assert row["success"] == 1
|
|
finally:
|
|
routing.model_store.remove("fb-primary-route")
|
|
routing.model_store.remove("fb-backup-route")
|
|
|
|
|
|
class FakeChainClient_openai_gateway:
|
|
def __init__(self, *a, **k):
|
|
self.calls = []
|
|
|
|
def build_request(self, method, url, headers=None, json=None, content=None):
|
|
return FakeRequest(method, url, json)
|
|
|
|
async def send(self, request, stream=False):
|
|
self.calls.append((request.url, request.json_body))
|
|
body = request.json_body or {}
|
|
model = body.get("model")
|
|
if model in ("chain-a-target", "chain-b-target"):
|
|
return FakeResp_openai_gateway(status=500, payload={"error": "boom"})
|
|
return FakeResp_openai_gateway(
|
|
payload={
|
|
"id": "x",
|
|
"model": model,
|
|
"choices": [{"message": {"content": "chain ok"}}],
|
|
}
|
|
)
|
|
|
|
async def aclose(self):
|
|
pass
|
|
|
|
|
|
def test_chat_walks_fallback_chain_past_failures(local_db, monkeypatch):
|
|
from devplacepy.services.openai_gateway import routing
|
|
|
|
monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeChainClient_openai_gateway)
|
|
routing.model_store.set(
|
|
routing.ModelRouteIn(source_model="chain-c", target_model="chain-c-target")
|
|
)
|
|
routing.model_store.set(
|
|
routing.ModelRouteIn(
|
|
source_model="chain-b",
|
|
target_model="chain-b-target",
|
|
fallback_model="chain-c",
|
|
)
|
|
)
|
|
routing.model_store.set(
|
|
routing.ModelRouteIn(
|
|
source_model="chain-a",
|
|
target_model="chain-a-target",
|
|
fallback_model="chain-b",
|
|
)
|
|
)
|
|
try:
|
|
svc = GatewayService()
|
|
cfg = svc.effective_config()
|
|
cfg["gateway_vision_enabled"] = False
|
|
cfg["gateway_max_retries"] = 0
|
|
rt = svc.runtime()
|
|
response = run_async(
|
|
rt.handle_chat(
|
|
{"model": "chain-a", "messages": [{"role": "user", "content": "hi"}]},
|
|
cfg,
|
|
("guest", "chain_walk"),
|
|
"test",
|
|
"default",
|
|
)
|
|
)
|
|
assert response.status_code == 200
|
|
assert [c[1]["model"] for c in rt._client.calls] == [
|
|
"chain-a-target",
|
|
"chain-b-target",
|
|
"chain-c-target",
|
|
]
|
|
row = get_table("gateway_usage_ledger").find_one(owner_id="chain_walk")
|
|
assert row["model"] == "chain-c-target"
|
|
assert row["fallback_used_route"] == "chain-c"
|
|
finally:
|
|
routing.model_store.remove("chain-a")
|
|
routing.model_store.remove("chain-b")
|
|
routing.model_store.remove("chain-c")
|
|
|
|
|
|
def test_chat_preroutes_around_unhealthy_models(local_db, monkeypatch):
|
|
from devplacepy.services.openai_gateway import health, routing
|
|
|
|
monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeClient_openai_gateway)
|
|
routing.model_store.set(
|
|
routing.ModelRouteIn(source_model="pre-good", target_model="good-target")
|
|
)
|
|
routing.model_store.set(
|
|
routing.ModelRouteIn(
|
|
source_model="pre-bad",
|
|
target_model="bad-target",
|
|
fallback_model="pre-good",
|
|
)
|
|
)
|
|
health.record_health(
|
|
"pre-bad", health.ProbeResult(False, "http_500", "boom", 1.0)
|
|
)
|
|
try:
|
|
svc = GatewayService()
|
|
cfg = svc.effective_config()
|
|
cfg["gateway_vision_enabled"] = False
|
|
rt = svc.runtime()
|
|
logged = []
|
|
response = run_async(
|
|
rt.handle_chat(
|
|
{"model": "pre-bad", "messages": [{"role": "user", "content": "hi"}]},
|
|
cfg,
|
|
("guest", "preroute"),
|
|
"test",
|
|
"default",
|
|
log=logged.append,
|
|
)
|
|
)
|
|
assert response.status_code == 200
|
|
assert len(rt._client.calls) == 1
|
|
assert rt._client.calls[0][1]["model"] == "good-target"
|
|
assert any("is unhealthy, starting on" in line for line in logged)
|
|
row = get_table("gateway_usage_ledger").find_one(owner_id="preroute")
|
|
assert row["requested_model"] == "pre-bad"
|
|
assert row["model"] == "good-target"
|
|
finally:
|
|
routing.model_store.remove("pre-bad")
|
|
routing.model_store.remove("pre-good")
|
|
|
|
|
|
def test_chat_tries_original_when_chain_is_exhausted(local_db, monkeypatch):
|
|
from devplacepy.services.openai_gateway import health, routing
|
|
|
|
monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeClient_openai_gateway)
|
|
routing.model_store.set(
|
|
routing.ModelRouteIn(
|
|
source_model="exh-fb",
|
|
target_model="exh-fb-target",
|
|
fallback_model="exh-primary",
|
|
)
|
|
)
|
|
routing.model_store.set(
|
|
routing.ModelRouteIn(
|
|
source_model="exh-primary",
|
|
target_model="exh-primary-target",
|
|
fallback_model="exh-fb",
|
|
)
|
|
)
|
|
health.record_health(
|
|
"exh-primary", health.ProbeResult(False, "http_500", "boom", 1.0)
|
|
)
|
|
health.record_health(
|
|
"exh-fb", health.ProbeResult(False, "http_500", "boom", 1.0)
|
|
)
|
|
try:
|
|
svc = GatewayService()
|
|
cfg = svc.effective_config()
|
|
cfg["gateway_vision_enabled"] = False
|
|
rt = svc.runtime()
|
|
response = run_async(
|
|
rt.handle_chat(
|
|
{"model": "exh-primary", "messages": [{"role": "user", "content": "hi"}]},
|
|
cfg,
|
|
("guest", "exhausted"),
|
|
"test",
|
|
"default",
|
|
)
|
|
)
|
|
assert response.status_code == 200
|
|
assert [c[1]["model"] for c in rt._client.calls] == ["exh-primary-target"]
|
|
finally:
|
|
routing.model_store.remove("exh-primary")
|
|
routing.model_store.remove("exh-fb")
|
|
|
|
|
|
def test_models_endpoint_hides_unhealthy_routes(local_db):
|
|
from devplacepy.services.openai_gateway import health, routing
|
|
|
|
routing.model_store.set(
|
|
routing.ModelRouteIn(source_model="hide-me", target_model="v/hide")
|
|
)
|
|
health.record_health(
|
|
"hide-me", health.ProbeResult(False, "http_500", "boom", 1.0)
|
|
)
|
|
try:
|
|
svc = GatewayService()
|
|
payload = json.loads(svc._models_response().body.decode())
|
|
assert "hide-me" not in {m["id"] for m in payload["data"]}
|
|
health.record_health(
|
|
"hide-me", health.ProbeResult(True, "ok", "", 1.0)
|
|
)
|
|
payload = json.loads(svc._models_response().body.decode())
|
|
assert "hide-me" in {m["id"] for m in payload["data"]}
|
|
finally:
|
|
routing.model_store.remove("hide-me")
|
|
|
|
|
|
class FakeContextLengthClient_openai_gateway:
|
|
def __init__(self, *a, **k):
|
|
self.calls = []
|
|
|
|
def build_request(self, method, url, headers=None, json=None, content=None):
|
|
return FakeRequest(method, url, json)
|
|
|
|
async def send(self, request, stream=False):
|
|
self.calls.append((request.url, request.json_body))
|
|
body = request.json_body or {}
|
|
model = body.get("model")
|
|
if model == "ctx-primary-target":
|
|
return FakeResp_openai_gateway(
|
|
status=400,
|
|
payload={"error": {"message": "maximum context length exceeded"}},
|
|
)
|
|
return FakeResp_openai_gateway(
|
|
payload={
|
|
"id": "x",
|
|
"model": model,
|
|
"choices": [{"message": {"content": "fallback ok"}}],
|
|
}
|
|
)
|
|
|
|
async def aclose(self):
|
|
pass
|
|
|
|
|
|
def test_chat_falls_back_on_a_context_length_error(local_db, monkeypatch):
|
|
from devplacepy.services.openai_gateway import routing
|
|
|
|
monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeContextLengthClient_openai_gateway)
|
|
routing.model_store.set(
|
|
routing.ModelRouteIn(source_model="ctx-backup-route", target_model="ctx-backup-target")
|
|
)
|
|
routing.model_store.set(
|
|
routing.ModelRouteIn(
|
|
source_model="ctx-primary-route",
|
|
target_model="ctx-primary-target",
|
|
fallback_model="ctx-backup-route",
|
|
)
|
|
)
|
|
try:
|
|
svc = GatewayService()
|
|
cfg = svc.effective_config()
|
|
cfg["gateway_vision_enabled"] = False
|
|
cfg["gateway_max_retries"] = 0
|
|
rt = svc.runtime()
|
|
response = run_async(
|
|
rt.handle_chat(
|
|
{"model": "ctx-primary-route", "messages": [{"role": "user", "content": "hi"}]},
|
|
cfg,
|
|
("guest", "context_length_fallback_success"),
|
|
"test",
|
|
"default",
|
|
)
|
|
)
|
|
assert response.status_code == 200
|
|
assert rt._client.calls[0][1]["model"] == "ctx-primary-target"
|
|
assert rt._client.calls[-1][1]["model"] == "ctx-backup-target"
|
|
row = get_table("gateway_usage_ledger").find_one(
|
|
owner_id="context_length_fallback_success"
|
|
)
|
|
assert row["success"] == 1
|
|
assert row["fallback_used_route"] == "ctx-backup-route"
|
|
finally:
|
|
routing.model_store.remove("ctx-primary-route")
|
|
routing.model_store.remove("ctx-backup-route")
|
|
|
|
|
|
def test_chat_skips_a_too_small_primary_and_goes_straight_to_fallback(local_db, monkeypatch):
|
|
from devplacepy.services.openai_gateway import routing
|
|
|
|
monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeContextLengthClient_openai_gateway)
|
|
routing.model_store.set(
|
|
routing.ModelRouteIn(
|
|
source_model="ctx-precheck-backup",
|
|
target_model="ctx-backup-target",
|
|
context_window=1_000_000,
|
|
)
|
|
)
|
|
routing.model_store.set(
|
|
routing.ModelRouteIn(
|
|
source_model="ctx-precheck-primary",
|
|
target_model="ctx-primary-target",
|
|
context_window=50,
|
|
fallback_model="ctx-precheck-backup",
|
|
)
|
|
)
|
|
try:
|
|
svc = GatewayService()
|
|
cfg = svc.effective_config()
|
|
cfg["gateway_vision_enabled"] = False
|
|
cfg["gateway_max_retries"] = 0
|
|
rt = svc.runtime()
|
|
long_message = "word " * 2000
|
|
response = run_async(
|
|
rt.handle_chat(
|
|
{
|
|
"model": "ctx-precheck-primary",
|
|
"messages": [{"role": "user", "content": long_message}],
|
|
},
|
|
cfg,
|
|
("guest", "context_precheck_skips_primary"),
|
|
"test",
|
|
"default",
|
|
)
|
|
)
|
|
assert response.status_code == 200
|
|
# Only the fallback was ever called - the primary was known too small.
|
|
assert len(rt._client.calls) == 1
|
|
assert rt._client.calls[0][1]["model"] == "ctx-backup-target"
|
|
row = get_table("gateway_usage_ledger").find_one(
|
|
owner_id="context_precheck_skips_primary"
|
|
)
|
|
assert row["fallback_used_route"] == "ctx-precheck-backup"
|
|
finally:
|
|
routing.model_store.remove("ctx-precheck-primary")
|
|
routing.model_store.remove("ctx-precheck-backup")
|
|
|
|
|
|
def test_chat_falls_back_via_molodetz_when_the_client_sends_an_unrouted_model_name(
|
|
local_db, monkeypatch
|
|
):
|
|
from devplacepy.services.openai_gateway import routing
|
|
|
|
original = routing.model_store.get("molodetz")
|
|
routing.model_store.set(
|
|
routing.ModelRouteIn(source_model="fb-molodetz-backup", target_model="backup-target")
|
|
)
|
|
routing.model_store.set(
|
|
routing.ModelRouteIn(
|
|
source_model="molodetz",
|
|
target_model="primary-target",
|
|
fallback_model="fb-molodetz-backup",
|
|
)
|
|
)
|
|
try:
|
|
svc = GatewayService()
|
|
cfg = svc.effective_config()
|
|
cfg["gateway_vision_enabled"] = False
|
|
cfg["gateway_max_retries"] = 0
|
|
cfg["gateway_force_model"] = False
|
|
base_default_model = cfg["gateway_model"]
|
|
|
|
class FakeUnroutedPrimaryFailsClient:
|
|
def __init__(self, *a, **k):
|
|
self.calls = []
|
|
|
|
def build_request(self, method, url, headers=None, json=None, content=None):
|
|
return FakeRequest(method, url, json)
|
|
|
|
async def send(self, request, stream=False):
|
|
self.calls.append((request.url, request.json_body))
|
|
body = request.json_body or {}
|
|
model = body.get("model")
|
|
if model == base_default_model:
|
|
return FakeResp_openai_gateway(
|
|
status=402, payload={"error": {"message": "Insufficient Balance"}}
|
|
)
|
|
return FakeResp_openai_gateway(
|
|
payload={
|
|
"id": "x",
|
|
"model": model,
|
|
"choices": [{"message": {"content": "fallback ok"}}],
|
|
}
|
|
)
|
|
|
|
async def aclose(self):
|
|
pass
|
|
|
|
monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeUnroutedPrimaryFailsClient)
|
|
rt = svc.runtime()
|
|
# The client sends a model name that matches no configured route at all
|
|
# (as a caller like dpc does, which is never told the exact "molodetz"
|
|
# routing alias) - the primary call falls through to the flat/global
|
|
# default model (no route overlay applies), but a failure there must
|
|
# still consult molodetz's own configured fallback, not silently skip
|
|
# it just because the client's literal string never matched anything.
|
|
response = run_async(
|
|
rt.handle_chat(
|
|
{
|
|
"model": "some-client-side-model-name",
|
|
"messages": [{"role": "user", "content": "hi"}],
|
|
},
|
|
cfg,
|
|
("guest", "fallback_chat_unrouted_client_model"),
|
|
"test",
|
|
"default",
|
|
)
|
|
)
|
|
assert response.status_code == 200
|
|
assert rt._client.calls[0][1]["model"] == base_default_model
|
|
assert rt._client.calls[-1][1]["model"] == "backup-target"
|
|
row = get_table("gateway_usage_ledger").find_one(
|
|
owner_id="fallback_chat_unrouted_client_model"
|
|
)
|
|
assert row is not None
|
|
assert row["requested_model"] == "some-client-side-model-name"
|
|
assert row["model"] == "backup-target"
|
|
assert row["success"] == 1
|
|
finally:
|
|
routing.model_store.remove("fb-molodetz-backup")
|
|
if original is not None:
|
|
routing.model_store.set(
|
|
routing.ModelRouteIn(
|
|
**{
|
|
field: getattr(original, field)
|
|
for field in routing.ModelRouteIn.model_fields
|
|
}
|
|
)
|
|
)
|
|
else:
|
|
routing.model_store.remove("molodetz")
|
|
|
|
|
|
def test_chat_returns_the_fallback_failure_when_both_models_fail(local_db, monkeypatch):
|
|
from devplacepy.services.openai_gateway import routing
|
|
|
|
monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeAlwaysFailClient_openai_gateway)
|
|
routing.model_store.set(
|
|
routing.ModelRouteIn(source_model="fb-backup-route2", target_model="backup-target2")
|
|
)
|
|
routing.model_store.set(
|
|
routing.ModelRouteIn(
|
|
source_model="fb-primary-route2",
|
|
target_model="primary-target2",
|
|
fallback_model="fb-backup-route2",
|
|
)
|
|
)
|
|
try:
|
|
svc = GatewayService()
|
|
cfg = svc.effective_config()
|
|
cfg["gateway_vision_enabled"] = False
|
|
cfg["gateway_max_retries"] = 0
|
|
rt = svc.runtime()
|
|
response = run_async(
|
|
rt.handle_chat(
|
|
{"model": "fb-primary-route2", "messages": [{"role": "user", "content": "hi"}]},
|
|
cfg,
|
|
("guest", "fallback_chat_both_fail"),
|
|
"test",
|
|
"default",
|
|
)
|
|
)
|
|
assert response.status_code == 500
|
|
assert len(rt._client.calls) == 2
|
|
assert rt._client.calls[0][1]["model"] == "primary-target2"
|
|
assert rt._client.calls[1][1]["model"] == "backup-target2"
|
|
finally:
|
|
routing.model_store.remove("fb-primary-route2")
|
|
routing.model_store.remove("fb-backup-route2")
|
|
|
|
|
|
def test_chat_without_a_fallback_never_makes_a_second_call(local_db, monkeypatch):
|
|
from devplacepy.services.openai_gateway import routing
|
|
|
|
monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeAlwaysFailClient_openai_gateway)
|
|
routing.model_store.set(
|
|
routing.ModelRouteIn(source_model="fb-no-fallback", target_model="no-fallback-target")
|
|
)
|
|
try:
|
|
svc = GatewayService()
|
|
cfg = svc.effective_config()
|
|
cfg["gateway_vision_enabled"] = False
|
|
cfg["gateway_max_retries"] = 0
|
|
rt = svc.runtime()
|
|
run_async(
|
|
rt.handle_chat(
|
|
{"model": "fb-no-fallback", "messages": [{"role": "user", "content": "hi"}]},
|
|
cfg,
|
|
("guest", "no_fallback_configured"),
|
|
"test",
|
|
"default",
|
|
)
|
|
)
|
|
assert len(rt._client.calls) == 1
|
|
finally:
|
|
routing.model_store.remove("fb-no-fallback")
|
|
|
|
|
|
def test_embeddings_unknown_model_falls_back_to_default(local_db, monkeypatch):
|
|
monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeEmbedClient_openai_gateway)
|
|
monkeypatch.setenv("OPENROUTER_API_KEY", "test-embed-key")
|
|
svc = GatewayService()
|
|
cfg = svc.effective_config()
|
|
cfg["gateway_force_model"] = False
|
|
cfg["gateway_embed_enabled"] = True
|
|
cfg["gateway_embed_model"] = "qwen/qwen3-embedding-8b"
|
|
rt = svc.runtime()
|
|
run_async(
|
|
rt.handle_embeddings(
|
|
{"model": "nonexistent-embed-model", "input": "hello"},
|
|
cfg,
|
|
("guest", "fallback_test_embed"),
|
|
"test",
|
|
"default",
|
|
)
|
|
)
|
|
assert rt._client.calls[-1][1]["model"] == "qwen/qwen3-embedding-8b"
|
|
|
|
|
|
def test_images_unknown_model_falls_back_to_default(local_db, monkeypatch):
|
|
monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeImageClient_openai_gateway)
|
|
svc = GatewayService()
|
|
cfg = svc.effective_config()
|
|
cfg["gateway_force_model"] = False
|
|
cfg["gateway_image_enabled"] = True
|
|
cfg["gateway_image_model"] = "black-forest-labs/flux.2-pro"
|
|
rt = svc.runtime()
|
|
run_async(
|
|
rt.handle_images(
|
|
{"model": "nonexistent-image-model", "prompt": "test badge"},
|
|
cfg,
|
|
("guest", "fallback_test_img"),
|
|
"test",
|
|
"default",
|
|
)
|
|
)
|
|
assert rt._client.calls[-1][1]["model"] == "black-forest-labs/flux.2-pro"
|
|
|
|
|
|
class FakeClientWithUsage_openai_gateway(FakeClient_openai_gateway):
|
|
async def send(self, request, stream=False):
|
|
self.calls.append((request.url, request.json_body))
|
|
body = request.json_body or {}
|
|
return FakeResp_openai_gateway(
|
|
payload={
|
|
"id": "x",
|
|
"model": body.get("model"),
|
|
"choices": [{"message": {"content": "hi there"}}],
|
|
"usage": {"prompt_tokens": 7, "completion_tokens": 3, "total_tokens": 10},
|
|
}
|
|
)
|
|
|
|
|
|
def test_ollama_stream_does_not_send_stream_options(local_db, monkeypatch):
|
|
monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeClient_openai_gateway)
|
|
svc = GatewayService()
|
|
cfg = svc.effective_config()
|
|
cfg["gateway_upstream_url"] = "http://127.0.0.1:11434/api/chat"
|
|
rt = svc.runtime()
|
|
run_async(
|
|
rt.handle_chat(
|
|
{
|
|
"messages": [{"role": "user", "content": "hi"}],
|
|
"stream": True,
|
|
"stream_options": {"include_usage": True},
|
|
},
|
|
cfg,
|
|
("guest", "ollama_stream_opts"),
|
|
"test",
|
|
"default",
|
|
)
|
|
)
|
|
upstream_payload = rt._client.calls[-1][1]
|
|
assert upstream_payload["stream"] is True
|
|
assert "stream_options" not in upstream_payload
|
|
assert upstream_payload["think"] is False
|
|
|
|
|
|
def test_ollama_explicit_dialect_overrides_url(local_db, monkeypatch):
|
|
monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeClient_openai_gateway)
|
|
svc = GatewayService()
|
|
cfg = svc.effective_config()
|
|
cfg["gateway_upstream_url"] = "https://ai.example.com/v1/chat/completions"
|
|
cfg["gateway_thinking_dialect"] = "ollama"
|
|
rt = svc.runtime()
|
|
run_async(
|
|
rt.handle_chat(
|
|
{"messages": [{"role": "user", "content": "hi"}]},
|
|
cfg,
|
|
("guest", "ollama_dialect_override"),
|
|
"test",
|
|
"default",
|
|
)
|
|
)
|
|
body = rt._client.calls[-1][1]
|
|
assert body["think"] is False
|
|
assert "thinking" not in body
|
|
|
|
|
|
def test_client_model_allowed_when_allow_client_model_on(local_db, monkeypatch):
|
|
monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeClient_openai_gateway)
|
|
svc = GatewayService()
|
|
cfg = svc.effective_config()
|
|
cfg["gateway_force_model"] = True
|
|
cfg["gateway_allow_client_model"] = True
|
|
cfg["gateway_model"] = "deepseek-chat"
|
|
rt = svc.runtime()
|
|
run_async(
|
|
rt.handle_chat(
|
|
{"model": "llama3.2", "messages": [{"role": "user", "content": "hi"}]},
|
|
cfg,
|
|
("guest", "client_model"),
|
|
"test",
|
|
"default",
|
|
)
|
|
)
|
|
assert rt._client.calls[-1][1]["model"] == "llama3.2"
|
|
|
|
|
|
def test_molodetz_alias_still_remapped_with_allow_client_model(local_db, monkeypatch):
|
|
from devplacepy.services.openai_gateway import routing
|
|
|
|
monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeClient_openai_gateway)
|
|
original = routing.model_store.get("molodetz")
|
|
routing.model_store.remove("molodetz")
|
|
try:
|
|
svc = GatewayService()
|
|
cfg = svc.effective_config()
|
|
cfg["gateway_force_model"] = True
|
|
cfg["gateway_allow_client_model"] = True
|
|
cfg["gateway_model"] = "deepseek-chat"
|
|
rt = svc.runtime()
|
|
run_async(
|
|
rt.handle_chat(
|
|
{"model": "molodetz", "messages": [{"role": "user", "content": "hi"}]},
|
|
cfg,
|
|
("guest", "molodetz_alias"),
|
|
"test",
|
|
"default",
|
|
)
|
|
)
|
|
assert rt._client.calls[-1][1]["model"] == "deepseek-chat"
|
|
finally:
|
|
if original is not None:
|
|
routing.model_store.set(
|
|
routing.ModelRouteIn(
|
|
**{
|
|
field: getattr(original, field)
|
|
for field in routing.ModelRouteIn.model_fields
|
|
}
|
|
)
|
|
)
|
|
|
|
|
|
def test_stream_forwarded_to_upstream_with_forced_include_usage(local_db, monkeypatch):
|
|
monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeClient_openai_gateway)
|
|
svc = GatewayService()
|
|
cfg = svc.effective_config()
|
|
rt = svc.runtime()
|
|
run_async(
|
|
rt.handle_chat(
|
|
{
|
|
"messages": [{"role": "user", "content": "hi"}],
|
|
"stream": True,
|
|
"stream_options": {"include_usage": False},
|
|
},
|
|
cfg,
|
|
("guest", "test"),
|
|
"test",
|
|
"default",
|
|
)
|
|
)
|
|
upstream_payload = rt._client.calls[-1][1]
|
|
assert upstream_payload["stream"] is True
|
|
assert upstream_payload["stream_options"] == {"include_usage": True}
|
|
assert upstream_payload["thinking"] == {"type": "disabled"}
|
|
assert "think" not in upstream_payload
|
|
assert "reasoning" not in upstream_payload
|
|
|
|
|
|
def test_non_stream_has_no_stream_options_upstream(local_db, monkeypatch):
|
|
monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeClient_openai_gateway)
|
|
svc = GatewayService()
|
|
cfg = svc.effective_config()
|
|
rt = svc.runtime()
|
|
run_async(
|
|
rt.handle_chat(
|
|
{
|
|
"messages": [{"role": "user", "content": "hi"}],
|
|
"stream_options": {"include_usage": True},
|
|
},
|
|
cfg,
|
|
("guest", "test"),
|
|
"test",
|
|
"default",
|
|
)
|
|
)
|
|
upstream_payload = rt._client.calls[-1][1]
|
|
assert "stream_options" not in upstream_payload
|
|
assert upstream_payload["stream"] is False
|
|
|
|
|
|
def test_thinking_disabled_by_default(local_db, monkeypatch):
|
|
monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeClient_openai_gateway)
|
|
svc = GatewayService()
|
|
cfg = svc.effective_config()
|
|
rt = svc.runtime()
|
|
run_async(
|
|
rt.handle_chat(
|
|
{"messages": [{"role": "user", "content": "hi"}]},
|
|
cfg,
|
|
("guest", "test"),
|
|
"test",
|
|
"default",
|
|
)
|
|
)
|
|
body = rt._client.calls[-1][1]
|
|
assert body["thinking"] == {"type": "disabled"}
|
|
|
|
|
|
def test_client_think_true_enables_upstream_thinking(local_db, monkeypatch):
|
|
monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeClient_openai_gateway)
|
|
svc = GatewayService()
|
|
cfg = svc.effective_config()
|
|
rt = svc.runtime()
|
|
run_async(
|
|
rt.handle_chat(
|
|
{
|
|
"messages": [{"role": "user", "content": "hi"}],
|
|
"think": True,
|
|
},
|
|
cfg,
|
|
("guest", "test"),
|
|
"test",
|
|
"default",
|
|
)
|
|
)
|
|
body = rt._client.calls[-1][1]
|
|
assert body["thinking"] == {"type": "enabled"}
|
|
assert "think" not in body
|
|
|
|
|
|
def test_openrouter_upstream_gets_reasoning_none(local_db, monkeypatch):
|
|
monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeClient_openai_gateway)
|
|
svc = GatewayService()
|
|
cfg = svc.effective_config()
|
|
cfg["gateway_upstream_url"] = "https://openrouter.ai/api/v1/chat/completions"
|
|
rt = svc.runtime()
|
|
run_async(
|
|
rt.handle_chat(
|
|
{"messages": [{"role": "user", "content": "hi"}]},
|
|
cfg,
|
|
("guest", "test"),
|
|
"test",
|
|
"default",
|
|
)
|
|
)
|
|
body = rt._client.calls[-1][1]
|
|
assert body["reasoning"] == {"effort": "none"}
|
|
assert "thinking" not in body
|
|
|
|
|
|
def test_include_usage_emits_usage_chunk(local_db, monkeypatch):
|
|
monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeClientWithUsage_openai_gateway)
|
|
svc = GatewayService()
|
|
cfg = svc.effective_config()
|
|
rt = svc.runtime()
|
|
resp = run_async(
|
|
rt.handle_chat(
|
|
{
|
|
"messages": [{"role": "user", "content": "hi"}],
|
|
"stream": True,
|
|
"stream_options": {"include_usage": True},
|
|
},
|
|
cfg,
|
|
("guest", "test"),
|
|
"test",
|
|
"default",
|
|
)
|
|
)
|
|
|
|
async def drain():
|
|
out = []
|
|
async for chunk in resp.body_iterator:
|
|
out.append(chunk if isinstance(chunk, str) else chunk.decode())
|
|
return "".join(out)
|
|
|
|
body = run_async(drain())
|
|
assert '"usage"' in body
|
|
assert '"total_tokens": 10' in body
|
|
assert "[DONE]" in body
|
|
|
|
|
|
def test_no_usage_chunk_without_include_usage(local_db, monkeypatch):
|
|
monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeClientWithUsage_openai_gateway)
|
|
svc = GatewayService()
|
|
cfg = svc.effective_config()
|
|
rt = svc.runtime()
|
|
resp = run_async(
|
|
rt.handle_chat(
|
|
{"messages": [{"role": "user", "content": "hi"}], "stream": True},
|
|
cfg,
|
|
("guest", "test"),
|
|
"test",
|
|
"default",
|
|
)
|
|
)
|
|
|
|
async def drain():
|
|
out = []
|
|
async for chunk in resp.body_iterator:
|
|
out.append(chunk if isinstance(chunk, str) else chunk.decode())
|
|
return "".join(out)
|
|
|
|
body = run_async(drain())
|
|
assert '"usage"' not in body
|
|
assert "[DONE]" in body
|
|
|
|
|
|
def test_stream_headers_carry_no_cost_or_token_data(local_db, monkeypatch):
|
|
monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeClientWithUsage_openai_gateway)
|
|
svc = GatewayService()
|
|
cfg = svc.effective_config()
|
|
rt = svc.runtime()
|
|
resp = run_async(
|
|
rt.handle_chat(
|
|
{"messages": [{"role": "user", "content": "hi"}], "stream": True},
|
|
cfg,
|
|
("guest", "test"),
|
|
"test",
|
|
"my-app",
|
|
)
|
|
)
|
|
assert resp.headers["X-Gateway-Model"] == cfg["gateway_model"]
|
|
assert resp.headers["X-Gateway-Backend"] == "chat"
|
|
assert resp.headers["X-App-Reference"] == "my-app"
|
|
assert "x-gateway-cost-usd" not in {k.lower() for k in resp.headers}
|
|
assert "x-gateway-total-tokens" not in {k.lower() for k in resp.headers}
|
|
|
|
|
|
def test_stream_records_ttft_and_inter_token_in_ledger(local_db, monkeypatch):
|
|
monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeClientWithUsage_openai_gateway)
|
|
svc = GatewayService()
|
|
cfg = svc.effective_config()
|
|
rt = svc.runtime()
|
|
resp = run_async(
|
|
rt.handle_chat(
|
|
{
|
|
"messages": [{"role": "user", "content": "hi"}],
|
|
"stream": True,
|
|
"stream_options": {"include_usage": True},
|
|
},
|
|
cfg,
|
|
("guest", "ttft_probe"),
|
|
"test",
|
|
"default",
|
|
)
|
|
)
|
|
|
|
async def drain():
|
|
async for _ in resp.body_iterator:
|
|
pass
|
|
|
|
run_async(drain())
|
|
|
|
row = get_table("gateway_usage_ledger").find_one(owner_id="ttft_probe")
|
|
assert row is not None
|
|
assert row["backend"] == "chat"
|
|
assert row["stream_requested"] == 1
|
|
assert row["success"] == 1
|
|
assert row["total_tokens"] == 10
|
|
assert row["ttft_ms"] is not None and row["ttft_ms"] >= 0.0
|
|
assert row["inter_token_ms"] is not None and row["inter_token_ms"] >= 0.0
|
|
|
|
|
|
class FakeClientReasoning_openai_gateway(FakeClient_openai_gateway):
|
|
async def send(self, request, stream=False):
|
|
self.calls.append((request.url, request.json_body))
|
|
body = request.json_body or {}
|
|
return FakeResp_openai_gateway(
|
|
payload={
|
|
"id": "x",
|
|
"model": body.get("model"),
|
|
"choices": [{"message": {"content": "x", "reasoning": "aaaaa"}}],
|
|
"usage": {"prompt_tokens": 7, "completion_tokens": 3, "total_tokens": 10},
|
|
}
|
|
)
|
|
|
|
|
|
def test_ollama_reasoning_delta_is_counted_as_a_content_chunk(local_db, monkeypatch):
|
|
monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeClientReasoning_openai_gateway)
|
|
svc = GatewayService()
|
|
cfg = svc.effective_config()
|
|
rt = svc.runtime()
|
|
resp = run_async(
|
|
rt.handle_chat(
|
|
{
|
|
"messages": [{"role": "user", "content": "hi"}],
|
|
"stream": True,
|
|
"stream_options": {"include_usage": True},
|
|
},
|
|
cfg,
|
|
("guest", "ollama_reasoning_probe"),
|
|
"test",
|
|
"default",
|
|
)
|
|
)
|
|
|
|
async def drain():
|
|
async for _ in resp.body_iterator:
|
|
pass
|
|
|
|
run_async(drain())
|
|
|
|
row = get_table("gateway_usage_ledger").find_one(owner_id="ollama_reasoning_probe")
|
|
assert row is not None
|
|
assert row["inter_token_ms"] is not None and row["inter_token_ms"] >= 0.0
|
|
|
|
|
|
def test_stream_client_disconnect_records_failure_and_closes_upstream(local_db, monkeypatch):
|
|
monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeClient_openai_gateway)
|
|
svc = GatewayService()
|
|
cfg = svc.effective_config()
|
|
rt = svc.runtime()
|
|
resp = run_async(
|
|
rt.handle_chat(
|
|
{"messages": [{"role": "user", "content": "hi"}], "stream": True},
|
|
cfg,
|
|
("guest", "disconnect_probe"),
|
|
"test",
|
|
"default",
|
|
)
|
|
)
|
|
|
|
async def partial_then_close():
|
|
agen = resp.body_iterator
|
|
await agen.__anext__()
|
|
await agen.aclose()
|
|
|
|
run_async(partial_then_close())
|
|
|
|
row = get_table("gateway_usage_ledger").find_one(owner_id="disconnect_probe")
|
|
assert row is not None
|
|
assert row["success"] == 0
|
|
assert row["error_category"] == "client_disconnected"
|
|
|
|
|
|
def test_models_endpoint_publishes_molodetz(local_db):
|
|
from devplacepy.services.openai_gateway import routing
|
|
|
|
routing.seed_default_deepseek_routes()
|
|
try:
|
|
svc = GatewayService()
|
|
resp = svc._models_response()
|
|
payload = json.loads(resp.body.decode())
|
|
ids = {m["id"] for m in payload["data"]}
|
|
assert payload["object"] == "list"
|
|
assert {"molodetz", "molodetz-pro"} <= ids
|
|
for model in payload["data"]:
|
|
assert model["object"] == "model"
|
|
assert model["owned_by"] == "molodetz"
|
|
finally:
|
|
routing.model_store.remove("molodetz")
|
|
routing.model_store.remove("molodetz-pro")
|
|
|
|
|
|
# Upstream header collision policy: only X-Gateway-Model is ever trusted and
|
|
# forwarded from an upstream that happens to speak our own X-Gateway-* header
|
|
# convention (e.g. another DevPlace-style gateway). Every other figure
|
|
# (cost/tokens/latency/context/app-reference) must remain ours regardless of
|
|
# what an upstream claims. See usage.upstream_reported_model / CLAUDE.md.
|
|
|
|
|
|
class FakeClientUpstreamModelHeader_openai_gateway(FakeClient_openai_gateway):
|
|
async def send(self, request, stream=False):
|
|
self.calls.append((request.url, request.json_body))
|
|
body = request.json_body or {}
|
|
return FakeResp_openai_gateway(
|
|
payload={
|
|
"id": "x",
|
|
"model": body.get("model"),
|
|
"choices": [{"message": {"content": "hi there"}}],
|
|
},
|
|
extra_headers={"X-Gateway-Model": "upstream/served-model-x"},
|
|
)
|
|
|
|
|
|
class FakeClientHostileModelHeader_openai_gateway(FakeClient_openai_gateway):
|
|
async def send(self, request, stream=False):
|
|
self.calls.append((request.url, request.json_body))
|
|
body = request.json_body or {}
|
|
return FakeResp_openai_gateway(
|
|
payload={
|
|
"id": "x",
|
|
"model": body.get("model"),
|
|
"choices": [{"message": {"content": "hi there"}}],
|
|
},
|
|
extra_headers={"X-Gateway-Model": "evil\r\nX-Injected: true"},
|
|
)
|
|
|
|
|
|
class FakeEmbedClientUpstreamModelHeader_openai_gateway(FakeEmbedClient_openai_gateway):
|
|
async def send(self, request, stream=False):
|
|
self.calls.append((request.url, request.json_body))
|
|
body = request.json_body or {}
|
|
return FakeResp_openai_gateway(
|
|
payload={
|
|
"object": "list",
|
|
"model": body.get("model"),
|
|
"data": [{"object": "embedding", "index": 0, "embedding": [0.1, 0.2]}],
|
|
"usage": {"prompt_tokens": 5, "total_tokens": 5},
|
|
},
|
|
extra_headers={"X-Gateway-Model": "upstream/embed-served"},
|
|
)
|
|
|
|
|
|
class FakeImageClientUpstreamModelHeader_openai_gateway(FakeImageClient_openai_gateway):
|
|
async def send(self, request, stream=False):
|
|
self.calls.append((request.url, request.json_body))
|
|
body = request.json_body or {}
|
|
return FakeResp_openai_gateway(
|
|
payload={
|
|
"created": 1,
|
|
"model": body.get("model"),
|
|
"data": [{"b64_json": "aGVsbG8="}],
|
|
"usage": {"cost": 0.05},
|
|
},
|
|
extra_headers={"X-Gateway-Model": "upstream/image-served"},
|
|
)
|
|
|
|
|
|
class FakePassthroughClientUpstreamModelHeader_openai_gateway:
|
|
def __init__(self, *a, **k):
|
|
self.calls = []
|
|
|
|
def build_request(self, method, url, headers=None, json=None, content=None):
|
|
return FakeRequest(method, url, json)
|
|
|
|
async def send(self, request, stream=False):
|
|
self.calls.append((request.url, request.json_body))
|
|
return FakeResp_openai_gateway(
|
|
payload={"ok": True},
|
|
extra_headers={"X-Gateway-Model": "upstream/passthrough-served"},
|
|
)
|
|
|
|
async def aclose(self):
|
|
pass
|
|
|
|
|
|
def test_upstream_x_gateway_model_header_is_forwarded_for_chat(local_db, monkeypatch):
|
|
monkeypatch.setattr(
|
|
gwmod.httpx, "AsyncClient", FakeClientUpstreamModelHeader_openai_gateway
|
|
)
|
|
svc = GatewayService()
|
|
cfg = svc.effective_config()
|
|
cfg["gateway_force_model"] = True
|
|
cfg["gateway_model"] = "deepseek-chat"
|
|
rt = svc.runtime()
|
|
resp = run_async(
|
|
rt.handle_chat(
|
|
{"messages": [{"role": "user", "content": "hi"}]},
|
|
cfg,
|
|
("guest", "upstream_model_chat"),
|
|
"test",
|
|
"default",
|
|
)
|
|
)
|
|
assert resp.headers["X-Gateway-Model"] == "upstream/served-model-x"
|
|
row = get_table("gateway_usage_ledger").find_one(owner_id="upstream_model_chat")
|
|
assert row is not None
|
|
assert row["model"] == "deepseek-chat"
|
|
|
|
|
|
def test_upstream_x_gateway_model_header_is_forwarded_for_streaming_chat(
|
|
local_db, monkeypatch
|
|
):
|
|
monkeypatch.setattr(
|
|
gwmod.httpx, "AsyncClient", FakeClientUpstreamModelHeader_openai_gateway
|
|
)
|
|
svc = GatewayService()
|
|
cfg = svc.effective_config()
|
|
rt = svc.runtime()
|
|
resp = run_async(
|
|
rt.handle_chat(
|
|
{"messages": [{"role": "user", "content": "hi"}], "stream": True},
|
|
cfg,
|
|
("guest", "upstream_model_stream"),
|
|
"test",
|
|
"default",
|
|
)
|
|
)
|
|
assert resp.headers["X-Gateway-Model"] == "upstream/served-model-x"
|
|
assert resp.headers["X-Gateway-Backend"] == "chat"
|
|
|
|
async def drain():
|
|
async for _ in resp.body_iterator:
|
|
pass
|
|
|
|
run_async(drain())
|
|
|
|
|
|
def test_upstream_x_gateway_model_header_is_forwarded_for_embeddings(
|
|
local_db, monkeypatch
|
|
):
|
|
monkeypatch.setattr(
|
|
gwmod.httpx, "AsyncClient", FakeEmbedClientUpstreamModelHeader_openai_gateway
|
|
)
|
|
monkeypatch.setenv("OPENROUTER_API_KEY", "test-embed-key")
|
|
svc = GatewayService()
|
|
cfg = svc.effective_config()
|
|
cfg["gateway_embed_enabled"] = True
|
|
cfg["gateway_embed_model"] = "qwen/qwen3-embedding-8b"
|
|
rt = svc.runtime()
|
|
resp = run_async(
|
|
rt.handle_embeddings(
|
|
{"input": "hello"},
|
|
cfg,
|
|
("guest", "upstream_model_embed"),
|
|
"test",
|
|
"default",
|
|
)
|
|
)
|
|
assert resp.headers["X-Gateway-Model"] == "upstream/embed-served"
|
|
row = get_table("gateway_usage_ledger").find_one(owner_id="upstream_model_embed")
|
|
assert row is not None
|
|
assert row["model"] == "qwen/qwen3-embedding-8b"
|
|
|
|
|
|
def test_upstream_x_gateway_model_header_is_forwarded_for_images(local_db, monkeypatch):
|
|
monkeypatch.setattr(
|
|
gwmod.httpx, "AsyncClient", FakeImageClientUpstreamModelHeader_openai_gateway
|
|
)
|
|
svc = GatewayService()
|
|
cfg = svc.effective_config()
|
|
cfg["gateway_image_enabled"] = True
|
|
cfg["gateway_image_model"] = "black-forest-labs/flux.2-pro"
|
|
rt = svc.runtime()
|
|
resp = run_async(
|
|
rt.handle_images(
|
|
{"prompt": "badge"},
|
|
cfg,
|
|
("guest", "upstream_model_img"),
|
|
"test",
|
|
"default",
|
|
)
|
|
)
|
|
assert resp.headers["X-Gateway-Model"] == "upstream/image-served"
|
|
row = get_table("gateway_usage_ledger").find_one(owner_id="upstream_model_img")
|
|
assert row is not None
|
|
assert row["model"] == "black-forest-labs/flux.2-pro"
|
|
|
|
|
|
def test_upstream_x_gateway_model_header_is_forwarded_for_passthrough(
|
|
local_db, monkeypatch
|
|
):
|
|
monkeypatch.setattr(
|
|
gwmod.httpx, "AsyncClient", FakePassthroughClientUpstreamModelHeader_openai_gateway
|
|
)
|
|
svc = GatewayService()
|
|
cfg = svc.effective_config()
|
|
cfg["gateway_model"] = "deepseek-chat"
|
|
rt = svc.runtime()
|
|
resp = run_async(
|
|
rt.handle_passthrough(
|
|
"POST",
|
|
"responses",
|
|
"application/json",
|
|
b"{}",
|
|
cfg,
|
|
("guest", "upstream_model_passthrough"),
|
|
"test",
|
|
"default",
|
|
)
|
|
)
|
|
assert resp.headers["X-Gateway-Model"] == "upstream/passthrough-served"
|
|
row = get_table("gateway_usage_ledger").find_one(
|
|
owner_id="upstream_model_passthrough"
|
|
)
|
|
assert row is not None
|
|
assert row["model"] == "deepseek-chat"
|
|
|
|
|
|
def test_no_upstream_model_header_keeps_our_own_model(local_db, monkeypatch):
|
|
monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeClient_openai_gateway)
|
|
svc = GatewayService()
|
|
cfg = svc.effective_config()
|
|
cfg["gateway_force_model"] = True
|
|
cfg["gateway_model"] = "deepseek-chat"
|
|
rt = svc.runtime()
|
|
resp = run_async(
|
|
rt.handle_chat(
|
|
{"messages": [{"role": "user", "content": "hi"}]},
|
|
cfg,
|
|
("guest", "no_upstream_header"),
|
|
"test",
|
|
"default",
|
|
)
|
|
)
|
|
assert resp.headers["X-Gateway-Model"] == "deepseek-chat"
|
|
|
|
|
|
class FakeEmbeddedErrorClient_openai_gateway:
|
|
"""Mimics OpenRouter's documented behavior of answering 200 OK with the
|
|
failure embedded in the JSON body (openrouter.ai/docs/api-reference/errors)
|
|
instead of a non-2xx status."""
|
|
|
|
def __init__(self, *a, **k):
|
|
self.calls = []
|
|
|
|
def build_request(self, method, url, headers=None, json=None, content=None):
|
|
return FakeRequest(method, url, json)
|
|
|
|
async def send(self, request, stream=False):
|
|
self.calls.append((request.url, request.json_body))
|
|
body = request.json_body or {}
|
|
model = body.get("model")
|
|
if model == "primary-target":
|
|
return FakeResp_openai_gateway(
|
|
status=200,
|
|
payload={"error": {"message": "no provider available", "code": 502}},
|
|
)
|
|
return FakeResp_openai_gateway(
|
|
payload={
|
|
"id": "x",
|
|
"model": model,
|
|
"choices": [{"message": {"content": "fallback ok"}}],
|
|
}
|
|
)
|
|
|
|
async def aclose(self):
|
|
pass
|
|
|
|
|
|
def test_chat_falls_back_when_upstream_returns_200_with_an_embedded_error(
|
|
local_db, monkeypatch
|
|
):
|
|
from devplacepy.services.openai_gateway import routing
|
|
|
|
monkeypatch.setattr(
|
|
gwmod.httpx, "AsyncClient", FakeEmbeddedErrorClient_openai_gateway
|
|
)
|
|
routing.model_store.set(
|
|
routing.ModelRouteIn(source_model="eb-backup-route", target_model="backup-target")
|
|
)
|
|
routing.model_store.set(
|
|
routing.ModelRouteIn(
|
|
source_model="eb-primary-route",
|
|
target_model="primary-target",
|
|
fallback_model="eb-backup-route",
|
|
)
|
|
)
|
|
try:
|
|
svc = GatewayService()
|
|
cfg = svc.effective_config()
|
|
cfg["gateway_vision_enabled"] = False
|
|
cfg["gateway_max_retries"] = 0
|
|
rt = svc.runtime()
|
|
response = run_async(
|
|
rt.handle_chat(
|
|
{"model": "eb-primary-route", "messages": [{"role": "user", "content": "hi"}]},
|
|
cfg,
|
|
("guest", "embedded_error_chat"),
|
|
"test",
|
|
"default",
|
|
)
|
|
)
|
|
assert response.status_code == 200
|
|
assert rt._client.calls[0][1]["model"] == "primary-target"
|
|
assert rt._client.calls[-1][1]["model"] == "backup-target"
|
|
row = get_table("gateway_usage_ledger").find_one(owner_id="embedded_error_chat")
|
|
assert row is not None
|
|
assert row["requested_model"] == "eb-primary-route"
|
|
assert row["model"] == "backup-target"
|
|
assert row["success"] == 1
|
|
finally:
|
|
routing.model_store.remove("eb-primary-route")
|
|
routing.model_store.remove("eb-backup-route")
|
|
|
|
|
|
class FakeAlwaysEmbeddedErrorClient_openai_gateway:
|
|
def __init__(self, *a, **k):
|
|
self.calls = []
|
|
|
|
def build_request(self, method, url, headers=None, json=None, content=None):
|
|
return FakeRequest(method, url, json)
|
|
|
|
async def send(self, request, stream=False):
|
|
self.calls.append((request.url, request.json_body))
|
|
return FakeResp_openai_gateway(
|
|
status=200,
|
|
payload={"error": {"message": "no provider available", "code": 502}},
|
|
)
|
|
|
|
async def aclose(self):
|
|
pass
|
|
|
|
|
|
def test_chat_reports_502_when_upstream_returns_200_with_an_embedded_error_and_no_fallback(
|
|
local_db, monkeypatch
|
|
):
|
|
monkeypatch.setattr(
|
|
gwmod.httpx, "AsyncClient", FakeAlwaysEmbeddedErrorClient_openai_gateway
|
|
)
|
|
svc = GatewayService()
|
|
cfg = svc.effective_config()
|
|
cfg["gateway_vision_enabled"] = False
|
|
cfg["gateway_max_retries"] = 0
|
|
rt = svc.runtime()
|
|
response = run_async(
|
|
rt.handle_chat(
|
|
{"messages": [{"role": "user", "content": "hi"}]},
|
|
cfg,
|
|
("guest", "embedded_error_no_fallback"),
|
|
"test",
|
|
"default",
|
|
)
|
|
)
|
|
assert response.status_code == 502
|
|
row = get_table("gateway_usage_ledger").find_one(owner_id="embedded_error_no_fallback")
|
|
assert row is not None
|
|
assert row["success"] == 0
|
|
assert row["error_category"] == "upstream_error"
|
|
|
|
|
|
def test_ollama_dialect_sends_reasoning_effort_alongside_think(local_db, monkeypatch):
|
|
# Ollama's native /api/chat honors a boolean `think`, but its
|
|
# OpenAI-compatible /v1/chat/completions layer (what this gateway
|
|
# actually calls) does not - it maps reasoning_effort/reasoning
|
|
# instead (github.com/ollama/ollama issues #15288, #15293, #14820).
|
|
monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeClient_openai_gateway)
|
|
svc = GatewayService()
|
|
cfg = svc.effective_config()
|
|
cfg["gateway_upstream_url"] = "http://127.0.0.1:11434/v1/chat/completions"
|
|
rt = svc.runtime()
|
|
run_async(
|
|
rt.handle_chat(
|
|
{"messages": [{"role": "user", "content": "hi"}]},
|
|
cfg,
|
|
("guest", "ollama_reasoning_effort"),
|
|
"test",
|
|
"default",
|
|
)
|
|
)
|
|
body = rt._client.calls[-1][1]
|
|
assert body["think"] is False
|
|
assert body["reasoning_effort"] == "none"
|
|
|
|
|
|
def test_hostile_upstream_model_header_is_ignored_end_to_end(local_db, monkeypatch):
|
|
monkeypatch.setattr(
|
|
gwmod.httpx, "AsyncClient", FakeClientHostileModelHeader_openai_gateway
|
|
)
|
|
svc = GatewayService()
|
|
cfg = svc.effective_config()
|
|
cfg["gateway_force_model"] = True
|
|
cfg["gateway_model"] = "deepseek-chat"
|
|
rt = svc.runtime()
|
|
resp = run_async(
|
|
rt.handle_chat(
|
|
{"messages": [{"role": "user", "content": "hi"}]},
|
|
cfg,
|
|
("guest", "hostile_header"),
|
|
"test",
|
|
"default",
|
|
)
|
|
)
|
|
assert resp.headers["X-Gateway-Model"] == "deepseek-chat"
|
|
|
|
|
|
def test_bypass_preamble_skips_the_operator_preamble_when_allowed(local_db, monkeypatch):
|
|
monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeClient_openai_gateway)
|
|
svc = GatewayService()
|
|
cfg = svc.effective_config()
|
|
cfg["gateway_system_preamble"] = "OPERATOR-PREAMBLE-TEXT"
|
|
rt = svc.runtime()
|
|
run_async(
|
|
rt.handle_chat(
|
|
{
|
|
"messages": [
|
|
{"role": "system", "content": "Reply with strict JSON only."},
|
|
{"role": "user", "content": "hi"},
|
|
],
|
|
"bypass_preamble": True,
|
|
},
|
|
cfg,
|
|
("internal", "devii"),
|
|
"test",
|
|
"default",
|
|
bypass_allowed=True,
|
|
)
|
|
)
|
|
sent = rt._client.calls[-1][1]
|
|
system_message = sent["messages"][0]
|
|
assert system_message["role"] == "system"
|
|
assert "OPERATOR-PREAMBLE-TEXT" not in system_message["content"]
|
|
assert "Current date" in system_message["content"]
|
|
assert "Reply with strict JSON only." in system_message["content"]
|
|
assert "bypass_preamble" not in sent
|
|
|
|
|
|
def test_bypass_preamble_ignored_without_internal_credentials(local_db, monkeypatch):
|
|
monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeClient_openai_gateway)
|
|
svc = GatewayService()
|
|
cfg = svc.effective_config()
|
|
cfg["gateway_system_preamble"] = "OPERATOR-PREAMBLE-TEXT"
|
|
rt = svc.runtime()
|
|
run_async(
|
|
rt.handle_chat(
|
|
{"messages": [{"role": "user", "content": "hi"}], "bypass_preamble": True},
|
|
cfg,
|
|
("user", "someuser"),
|
|
"test",
|
|
"default",
|
|
bypass_allowed=False,
|
|
)
|
|
)
|
|
sent = rt._client.calls[-1][1]
|
|
system_message = sent["messages"][0]
|
|
assert "OPERATOR-PREAMBLE-TEXT" in system_message["content"]
|
|
assert "bypass_preamble" not in sent
|
|
|
|
|
|
def test_bypass_preamble_keeps_the_clients_own_system_content(local_db, monkeypatch):
|
|
monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeClient_openai_gateway)
|
|
svc = GatewayService()
|
|
cfg = svc.effective_config()
|
|
cfg["gateway_system_preamble"] = "OPERATOR-PREAMBLE-TEXT"
|
|
rt = svc.runtime()
|
|
run_async(
|
|
rt.handle_chat(
|
|
{
|
|
"messages": [
|
|
{"role": "system", "content": "Grade strictly. Reply JSON only."},
|
|
{"role": "user", "content": "hi"},
|
|
],
|
|
"bypass_preamble": True,
|
|
},
|
|
cfg,
|
|
("internal", "devii"),
|
|
"test",
|
|
"default",
|
|
bypass_allowed=True,
|
|
)
|
|
)
|
|
sent = rt._client.calls[-1][1]
|
|
system_message = sent["messages"][0]
|
|
assert "OPERATOR-PREAMBLE-TEXT" not in system_message["content"]
|
|
assert "Grade strictly. Reply JSON only." in system_message["content"]
|
|
|
|
|
|
def test_internal_bypass_allowed_requires_the_exact_internal_key(local_db):
|
|
set_setting("gateway_internal_key", "test-internal-secret")
|
|
try:
|
|
svc = GatewayService()
|
|
cfg = svc.effective_config()
|
|
assert (
|
|
svc.internal_bypass_allowed(
|
|
_make_request_openai_gateway(
|
|
headers={"X-Gateway-Internal-Key": "test-internal-secret"}
|
|
),
|
|
cfg,
|
|
)
|
|
is True
|
|
)
|
|
assert (
|
|
svc.internal_bypass_allowed(
|
|
_make_request_openai_gateway(
|
|
headers={"X-Gateway-Internal-Key": "wrong-secret"}
|
|
),
|
|
cfg,
|
|
)
|
|
is False
|
|
)
|
|
assert (
|
|
svc.internal_bypass_allowed(
|
|
_make_request_openai_gateway(
|
|
headers={"Authorization": "Bearer test-internal-secret"}
|
|
),
|
|
cfg,
|
|
)
|
|
is False
|
|
)
|
|
assert svc.internal_bypass_allowed(_make_request_openai_gateway(), cfg) is False
|
|
finally:
|
|
set_setting("gateway_internal_key", "")
|
|
|
|
|
|
class FakeResponsesResp_openai_gateway(FakeResp_openai_gateway):
|
|
def __init__(self, status=200, payload=None, events=None, **kw):
|
|
super().__init__(status=status, payload=payload, **kw)
|
|
self._events = events or []
|
|
|
|
async def aiter_lines(self):
|
|
for event in self._events:
|
|
yield f"data: {json.dumps(event)}"
|
|
yield "data: [DONE]"
|
|
|
|
|
|
def _resp_obj_openai_gateway(model):
|
|
return {
|
|
"id": "resp_1",
|
|
"object": "response",
|
|
"created_at": 7,
|
|
"status": "completed",
|
|
"model": model,
|
|
"output": [
|
|
{
|
|
"type": "message",
|
|
"role": "assistant",
|
|
"content": [{"type": "output_text", "text": "hi there"}],
|
|
}
|
|
],
|
|
"usage": {"input_tokens": 5, "output_tokens": 7, "total_tokens": 12},
|
|
}
|
|
|
|
|
|
def _resp_events_openai_gateway(model):
|
|
return [
|
|
{"type": "response.created", "response": {"id": "resp_1"}},
|
|
{"type": "response.output_text.delta", "delta": "hi "},
|
|
{"type": "response.output_text.delta", "delta": "there"},
|
|
{"type": "response.completed", "response": _resp_obj_openai_gateway(model)},
|
|
]
|
|
|
|
|
|
def _resp_tool_obj_openai_gateway(model):
|
|
obj = _resp_obj_openai_gateway(model)
|
|
obj["output"].append(
|
|
{
|
|
"type": "function_call",
|
|
"call_id": "call_7",
|
|
"name": "ext_bash",
|
|
"arguments": "{}",
|
|
}
|
|
)
|
|
return obj
|
|
|
|
|
|
def _resp_tool_events_openai_gateway(model):
|
|
return [
|
|
{"type": "response.created", "response": {"id": "resp_1"}},
|
|
{
|
|
"type": "response.output_item.added",
|
|
"output_index": 1,
|
|
"item": {
|
|
"type": "function_call",
|
|
"call_id": "call_7",
|
|
"name": "ext_bash",
|
|
"arguments": "",
|
|
},
|
|
},
|
|
{
|
|
"type": "response.function_call_arguments.delta",
|
|
"item_id": "call_7",
|
|
"output_index": 1,
|
|
"delta": "{}",
|
|
},
|
|
{
|
|
"type": "response.completed",
|
|
"response": _resp_tool_obj_openai_gateway(model),
|
|
},
|
|
]
|
|
|
|
|
|
class FakeResponsesClient_openai_gateway:
|
|
behavior = "ok"
|
|
|
|
def __init__(self, *a, **k):
|
|
self.calls = []
|
|
self.last_headers = {}
|
|
|
|
def build_request(self, method, url, headers=None, json=None, content=None):
|
|
return FakeRequest(method, url, json, headers)
|
|
|
|
async def send(self, request, stream=False):
|
|
self.calls.append((request.url, request.json_body))
|
|
self.last_headers = dict(request.headers or {})
|
|
body = request.json_body or {}
|
|
if self.behavior == "error-envelope":
|
|
return FakeResp_openai_gateway(
|
|
status=429,
|
|
payload={
|
|
"error": {
|
|
"message": "slow down",
|
|
"type": "rate_limit",
|
|
"code": "rate_limit_exceeded",
|
|
}
|
|
},
|
|
)
|
|
if self.behavior == "failed-object":
|
|
return FakeResponsesResp_openai_gateway(
|
|
payload={
|
|
"object": "response",
|
|
"status": "failed",
|
|
"error": {"message": "boom"},
|
|
}
|
|
)
|
|
if self.behavior == "attestation-denied":
|
|
return FakeResp_openai_gateway(
|
|
status=403,
|
|
payload={
|
|
"error": {
|
|
"message": "This API key is billed to a Muse Code "
|
|
"subscription. Muse Code subscriptions cover requests "
|
|
"from the Muse Code client only.",
|
|
"type": "permission_error",
|
|
"code": "client_attestation_denied",
|
|
}
|
|
},
|
|
)
|
|
model = body.get("model", "")
|
|
if self.behavior == "tool-call":
|
|
return FakeResponsesResp_openai_gateway(
|
|
payload=_resp_tool_obj_openai_gateway(model),
|
|
events=_resp_tool_events_openai_gateway(model),
|
|
)
|
|
return FakeResponsesResp_openai_gateway(
|
|
payload=_resp_obj_openai_gateway(model),
|
|
events=_resp_events_openai_gateway(model),
|
|
)
|
|
|
|
async def post(self, url, headers=None, json=None, timeout=None):
|
|
return await self.send(FakeRequest("POST", url, json), stream=False)
|
|
|
|
async def aclose(self):
|
|
pass
|
|
|
|
|
|
def _setup_responses_route_openai_gateway():
|
|
from devplacepy.services.openai_gateway import routing
|
|
|
|
routing.provider_store.set(
|
|
routing.ProviderIn(
|
|
name="gwmeta",
|
|
base_url="https://api.meta.ai/v1/responses",
|
|
api_key="sk-test",
|
|
protocol="responses",
|
|
)
|
|
)
|
|
routing.model_store.set(
|
|
routing.ModelRouteIn(
|
|
source_model="meta-chat",
|
|
provider="gwmeta",
|
|
target_model="muse-spark-1.3-contributor",
|
|
kind="chat",
|
|
)
|
|
)
|
|
|
|
|
|
def _teardown_responses_route_openai_gateway():
|
|
from devplacepy.services.openai_gateway import routing
|
|
|
|
routing.model_store.remove("meta-chat")
|
|
routing.provider_store.remove("gwmeta")
|
|
|
|
|
|
def _chat_body_openai_gateway(**over):
|
|
body = {
|
|
"model": "meta-chat",
|
|
"messages": [
|
|
{"role": "system", "content": "Be terse."},
|
|
{"role": "user", "content": "hi"},
|
|
],
|
|
}
|
|
body.update(over)
|
|
return body
|
|
|
|
|
|
def test_responses_protocol_transpiles_request_and_response(local_db, monkeypatch):
|
|
monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeResponsesClient_openai_gateway)
|
|
FakeResponsesClient_openai_gateway.behavior = "ok"
|
|
_setup_responses_route_openai_gateway()
|
|
try:
|
|
svc = GatewayService()
|
|
cfg = svc.effective_config()
|
|
cfg["gateway_vision_enabled"] = False
|
|
rt = svc.runtime()
|
|
resp = run_async(
|
|
rt.handle_chat(
|
|
_chat_body_openai_gateway(), cfg, ("guest", "test"), "test", "default"
|
|
)
|
|
)
|
|
url, sent = rt._client.calls[-1]
|
|
assert str(url) == "https://api.meta.ai/v1/responses"
|
|
assert sent["model"] == "muse-spark-1.3-contributor"
|
|
assert "messages" not in sent
|
|
assert "Be terse." in sent["instructions"]
|
|
assert sent["input"] == [
|
|
{"type": "message", "role": "user", "content": "hi"}
|
|
]
|
|
assert sent["reasoning"] == {"effort": "minimal"}
|
|
assert resp.status_code == 200
|
|
payload = json.loads(bytes(resp.body).decode())
|
|
assert payload["object"] == "chat.completion"
|
|
assert payload["choices"][0]["message"]["content"] == "hi there"
|
|
assert payload["choices"][0]["finish_reason"] == "stop"
|
|
assert payload["usage"]["prompt_tokens"] == 5
|
|
assert resp.headers["X-Gateway-Model"] == "muse-spark-1.3-contributor"
|
|
finally:
|
|
_teardown_responses_route_openai_gateway()
|
|
|
|
|
|
def test_muse_profile_headers_reach_the_wire(local_db, monkeypatch):
|
|
from devplacepy.services.openai_gateway import routing
|
|
|
|
monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeResponsesClient_openai_gateway)
|
|
FakeResponsesClient_openai_gateway.behavior = "ok"
|
|
routing.provider_store.set(
|
|
routing.ProviderIn(
|
|
name="wiremeta",
|
|
base_url="https://api.meta.ai/v1/responses",
|
|
api_key="sk-test",
|
|
protocol="responses",
|
|
client_profile="muse",
|
|
)
|
|
)
|
|
routing.model_store.set(
|
|
routing.ModelRouteIn(
|
|
source_model="wire-muse",
|
|
provider="wiremeta",
|
|
target_model="muse-spark",
|
|
kind="chat",
|
|
)
|
|
)
|
|
try:
|
|
svc = GatewayService()
|
|
cfg = svc.effective_config()
|
|
cfg["gateway_vision_enabled"] = False
|
|
rt = svc.runtime()
|
|
logged = []
|
|
resp = run_async(
|
|
rt.handle_chat(
|
|
{"model": "wire-muse", "messages": [{"role": "user", "content": "hi"}]},
|
|
cfg,
|
|
("guest", "test"),
|
|
"test",
|
|
"default",
|
|
log=logged.append,
|
|
)
|
|
)
|
|
assert resp.status_code == 200
|
|
sent_headers = rt._client.last_headers
|
|
assert sent_headers["X-Client-Id"] == "tbh:exec"
|
|
assert sent_headers["User-Agent"].startswith("muse-build/")
|
|
session = sent_headers["X-Tbh-Session-Id"]
|
|
assert session
|
|
assert sent_headers["X-Meta-Ai-Gateway-Session-Id"] == session
|
|
assert sent_headers["X-Tbh-Thread-Id"] == session
|
|
assert sent_headers["X-Tbh-Root-Session-Id"] == session
|
|
assert any("(client profile 'muse')" in line for line in logged)
|
|
finally:
|
|
routing.model_store.remove("wire-muse")
|
|
routing.provider_store.remove("wiremeta")
|
|
|
|
|
|
def _bash_tool_body_openai_gateway(**over):
|
|
body = _chat_body_openai_gateway(**over)
|
|
body["tools"] = [
|
|
{
|
|
"type": "function",
|
|
"function": {
|
|
"name": "bash",
|
|
"description": "run shell",
|
|
"parameters": {"type": "object", "properties": {}},
|
|
},
|
|
}
|
|
]
|
|
return body
|
|
|
|
|
|
def test_meta_tool_names_aliased_on_wire_and_restored(local_db, monkeypatch):
|
|
monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeResponsesClient_openai_gateway)
|
|
FakeResponsesClient_openai_gateway.behavior = "tool-call"
|
|
_setup_responses_route_openai_gateway()
|
|
try:
|
|
svc = GatewayService()
|
|
cfg = svc.effective_config()
|
|
cfg["gateway_vision_enabled"] = False
|
|
rt = svc.runtime()
|
|
logged = []
|
|
resp = run_async(
|
|
rt.handle_chat(
|
|
_bash_tool_body_openai_gateway(),
|
|
cfg,
|
|
("guest", "test"),
|
|
"test",
|
|
"default",
|
|
log=logged.append,
|
|
)
|
|
)
|
|
_, sent = rt._client.calls[-1]
|
|
assert sent["tools"][0]["name"] == "ext_bash"
|
|
assert any("bash->ext_bash" in line for line in logged)
|
|
assert resp.status_code == 200
|
|
payload = json.loads(bytes(resp.body).decode())
|
|
calls = payload["choices"][0]["message"]["tool_calls"]
|
|
assert calls[0]["function"]["name"] == "bash"
|
|
assert payload["choices"][0]["finish_reason"] == "tool_calls"
|
|
finally:
|
|
FakeResponsesClient_openai_gateway.behavior = "ok"
|
|
_teardown_responses_route_openai_gateway()
|
|
|
|
|
|
def test_meta_tool_names_restored_on_stream_chunks(local_db, monkeypatch):
|
|
monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeResponsesClient_openai_gateway)
|
|
FakeResponsesClient_openai_gateway.behavior = "tool-call"
|
|
_setup_responses_route_openai_gateway()
|
|
try:
|
|
svc = GatewayService()
|
|
cfg = svc.effective_config()
|
|
cfg["gateway_vision_enabled"] = False
|
|
rt = svc.runtime()
|
|
resp = run_async(
|
|
rt.handle_chat(
|
|
_bash_tool_body_openai_gateway(stream=True),
|
|
cfg,
|
|
("guest", "test"),
|
|
"test",
|
|
"default",
|
|
)
|
|
)
|
|
|
|
async def drain():
|
|
out = []
|
|
async for chunk in resp.body_iterator:
|
|
out.append(chunk if isinstance(chunk, str) else chunk.decode())
|
|
return "".join(out)
|
|
|
|
body = run_async(drain())
|
|
assert '"name": "bash"' in body
|
|
assert "ext_bash" not in body
|
|
assert "[DONE]" in body
|
|
finally:
|
|
FakeResponsesClient_openai_gateway.behavior = "ok"
|
|
_teardown_responses_route_openai_gateway()
|
|
|
|
|
|
def _setup_chat_meta_route_openai_gateway():
|
|
from devplacepy.services.openai_gateway import routing
|
|
|
|
routing.provider_store.set(
|
|
routing.ProviderIn(
|
|
name="gwmetacc",
|
|
base_url="https://api.meta.ai/v1/chat/completions",
|
|
api_key="sk-test",
|
|
)
|
|
)
|
|
routing.model_store.set(
|
|
routing.ModelRouteIn(
|
|
source_model="meta-chat-cc",
|
|
provider="gwmetacc",
|
|
target_model="muse-spark",
|
|
kind="chat",
|
|
)
|
|
)
|
|
|
|
|
|
def _teardown_chat_meta_route_openai_gateway():
|
|
from devplacepy.services.openai_gateway import routing
|
|
|
|
routing.model_store.remove("meta-chat-cc")
|
|
routing.provider_store.remove("gwmetacc")
|
|
|
|
|
|
def _tai_tool_body_openai_gateway(**over):
|
|
# Mirrors a real tai/detz turn: core tools are always sent, so "shell"
|
|
# (a Pi-verified gate trigger missing from the denylist until
|
|
# 2026-11-10) rides every request alongside the lazy "read_file".
|
|
body = _chat_body_openai_gateway(**over)
|
|
body["model"] = "meta-chat-cc"
|
|
body["tools"] = [
|
|
{
|
|
"type": "function",
|
|
"function": {
|
|
"name": "shell",
|
|
"description": "run a shell command",
|
|
"parameters": {"type": "object", "properties": {}},
|
|
},
|
|
},
|
|
{
|
|
"type": "function",
|
|
"function": {
|
|
"name": "read_file",
|
|
"description": "read a file",
|
|
"parameters": {"type": "object", "properties": {}},
|
|
},
|
|
},
|
|
{
|
|
"type": "function",
|
|
"function": {
|
|
"name": "web_fetch",
|
|
"description": "fetch a URL",
|
|
"parameters": {"type": "object", "properties": {}},
|
|
},
|
|
},
|
|
]
|
|
return body
|
|
|
|
|
|
class FakeChatMetaStreamResp_openai_gateway(FakeResp_openai_gateway):
|
|
def __init__(self, payload):
|
|
super().__init__(status=200, payload=payload)
|
|
self._payload = payload
|
|
|
|
async def aiter_lines(self):
|
|
calls = self._payload["choices"][0]["message"]["tool_calls"]
|
|
yield (
|
|
'data: {"id":"chatcmpl-x","object":"chat.completion.chunk",'
|
|
'"model":"m","choices":[{"index":0,"delta":{"role":"assistant"}}]}'
|
|
)
|
|
for pos, call in enumerate(calls):
|
|
name = call["function"]["name"]
|
|
yield (
|
|
'data: {"id":"chatcmpl-x","object":"chat.completion.chunk",'
|
|
'"model":"m","choices":[{"index":0,"delta":{"tool_calls":'
|
|
'[{"index":%d,"id":"call_%d","function":{"name":"%s",'
|
|
'"arguments":"{}"}}]}}]}' % (pos, pos, name)
|
|
)
|
|
yield (
|
|
'data: {"id":"chatcmpl-x","object":"chat.completion.chunk",'
|
|
'"model":"m","choices":[{"index":0,"delta":{},'
|
|
'"finish_reason":"tool_calls"}]}'
|
|
)
|
|
yield "data: [DONE]"
|
|
|
|
|
|
class FakeChatMetaClient_openai_gateway:
|
|
def __init__(self, *a, **k):
|
|
self.calls = []
|
|
|
|
def build_request(self, method, url, headers=None, json=None, content=None):
|
|
return FakeRequest(method, url, json, headers)
|
|
|
|
async def send(self, request, stream=False):
|
|
self.calls.append((request.url, request.json_body))
|
|
body = request.json_body or {}
|
|
wire_names = [
|
|
(tool.get("function") or {}).get("name")
|
|
for tool in body.get("tools") or []
|
|
]
|
|
payload = {
|
|
"id": "chatcmpl-x",
|
|
"object": "chat.completion",
|
|
"model": body.get("model"),
|
|
"choices": [
|
|
{
|
|
"index": 0,
|
|
"message": {
|
|
"role": "assistant",
|
|
"content": "",
|
|
"tool_calls": [
|
|
{
|
|
"id": "call_7",
|
|
"type": "function",
|
|
"function": {"name": name, "arguments": "{}"},
|
|
}
|
|
for name in wire_names
|
|
],
|
|
},
|
|
"finish_reason": "tool_calls",
|
|
}
|
|
],
|
|
"usage": {"prompt_tokens": 5, "completion_tokens": 2, "total_tokens": 7},
|
|
}
|
|
if stream:
|
|
return FakeChatMetaStreamResp_openai_gateway(payload)
|
|
return FakeResp_openai_gateway(
|
|
payload=payload, content=json.dumps(payload).encode()
|
|
)
|
|
|
|
async def post(self, url, headers=None, json=None, timeout=None):
|
|
return await self.send(FakeRequest("POST", url, json), stream=False)
|
|
|
|
async def aclose(self):
|
|
pass
|
|
|
|
|
|
def test_meta_tool_names_aliased_on_chat_completions_wire(local_db, monkeypatch):
|
|
monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeChatMetaClient_openai_gateway)
|
|
_setup_chat_meta_route_openai_gateway()
|
|
try:
|
|
svc = GatewayService()
|
|
cfg = svc.effective_config()
|
|
cfg["gateway_vision_enabled"] = False
|
|
rt = svc.runtime()
|
|
logged = []
|
|
resp = run_async(
|
|
rt.handle_chat(
|
|
_tai_tool_body_openai_gateway(),
|
|
cfg,
|
|
("guest", "test"),
|
|
"test",
|
|
"default",
|
|
log=logged.append,
|
|
)
|
|
)
|
|
_, sent = rt._client.calls[-1]
|
|
wire = [tool["function"]["name"] for tool in sent["tools"]]
|
|
assert wire == ["ext_shell", "ext_read_file", "ext_web_fetch"]
|
|
assert any("shell->ext_shell" in line for line in logged)
|
|
assert any("read_file->ext_read_file" in line for line in logged)
|
|
assert any("web_fetch->ext_web_fetch" in line for line in logged)
|
|
assert resp.status_code == 200
|
|
payload = json.loads(bytes(resp.body).decode())
|
|
calls = payload["choices"][0]["message"]["tool_calls"]
|
|
assert [call["function"]["name"] for call in calls] == [
|
|
"shell",
|
|
"read_file",
|
|
"web_fetch",
|
|
]
|
|
assert payload["choices"][0]["finish_reason"] == "tool_calls"
|
|
finally:
|
|
_teardown_chat_meta_route_openai_gateway()
|
|
|
|
|
|
def test_meta_tool_names_restored_on_chat_completions_stream(
|
|
local_db, monkeypatch
|
|
):
|
|
monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeChatMetaClient_openai_gateway)
|
|
_setup_chat_meta_route_openai_gateway()
|
|
try:
|
|
svc = GatewayService()
|
|
cfg = svc.effective_config()
|
|
cfg["gateway_vision_enabled"] = False
|
|
rt = svc.runtime()
|
|
resp = run_async(
|
|
rt.handle_chat(
|
|
_tai_tool_body_openai_gateway(stream=True),
|
|
cfg,
|
|
("guest", "test"),
|
|
"test",
|
|
"default",
|
|
)
|
|
)
|
|
|
|
async def drain():
|
|
out = []
|
|
async for chunk in resp.body_iterator:
|
|
out.append(chunk if isinstance(chunk, str) else chunk.decode())
|
|
return "".join(out)
|
|
|
|
body = run_async(drain())
|
|
_, sent = rt._client.calls[-1]
|
|
wire = [tool["function"]["name"] for tool in sent["tools"]]
|
|
assert wire == ["ext_shell", "ext_read_file", "ext_web_fetch"]
|
|
assert '"name": "shell"' in body
|
|
assert '"name": "read_file"' in body
|
|
assert '"name": "web_fetch"' in body
|
|
assert "ext_shell" not in body
|
|
assert "ext_read_file" not in body
|
|
assert "ext_web_fetch" not in body
|
|
assert "[DONE]" in body
|
|
finally:
|
|
_teardown_chat_meta_route_openai_gateway()
|
|
|
|
|
|
def test_responses_protocol_stream_relays_chat_chunks(local_db, monkeypatch):
|
|
monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeResponsesClient_openai_gateway)
|
|
FakeResponsesClient_openai_gateway.behavior = "ok"
|
|
_setup_responses_route_openai_gateway()
|
|
try:
|
|
svc = GatewayService()
|
|
cfg = svc.effective_config()
|
|
cfg["gateway_vision_enabled"] = False
|
|
rt = svc.runtime()
|
|
resp = run_async(
|
|
rt.handle_chat(
|
|
_chat_body_openai_gateway(stream=True),
|
|
cfg,
|
|
("guest", "test"),
|
|
"test",
|
|
"default",
|
|
)
|
|
)
|
|
|
|
async def drain():
|
|
out = []
|
|
async for chunk in resp.body_iterator:
|
|
out.append(chunk if isinstance(chunk, str) else chunk.decode())
|
|
return "".join(out)
|
|
|
|
body = run_async(drain())
|
|
assert "chat.completion.chunk" in body
|
|
assert "hi " in body and "there" in body
|
|
assert '"usage"' not in body
|
|
assert "[DONE]" in body
|
|
finally:
|
|
_teardown_responses_route_openai_gateway()
|
|
|
|
|
|
def test_responses_protocol_stream_relays_usage_when_asked(local_db, monkeypatch):
|
|
monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeResponsesClient_openai_gateway)
|
|
FakeResponsesClient_openai_gateway.behavior = "ok"
|
|
_setup_responses_route_openai_gateway()
|
|
try:
|
|
svc = GatewayService()
|
|
cfg = svc.effective_config()
|
|
cfg["gateway_vision_enabled"] = False
|
|
rt = svc.runtime()
|
|
resp = run_async(
|
|
rt.handle_chat(
|
|
_chat_body_openai_gateway(
|
|
stream=True, stream_options={"include_usage": True}
|
|
),
|
|
cfg,
|
|
("guest", "test"),
|
|
"test",
|
|
"default",
|
|
)
|
|
)
|
|
_, sent = rt._client.calls[-1]
|
|
assert "stream_options" not in sent
|
|
|
|
async def drain():
|
|
out = []
|
|
async for chunk in resp.body_iterator:
|
|
out.append(chunk if isinstance(chunk, str) else chunk.decode())
|
|
return "".join(out)
|
|
|
|
body = run_async(drain())
|
|
assert '"usage"' in body
|
|
assert '"prompt_tokens": 5' in body
|
|
usage_chunks = [
|
|
json.loads(line[len("data:"):])
|
|
for line in body.splitlines()
|
|
if line.startswith("data: {") and '"usage"' in line
|
|
]
|
|
assert len(usage_chunks) == 1
|
|
assert usage_chunks[0]["choices"] == []
|
|
finally:
|
|
_teardown_responses_route_openai_gateway()
|
|
|
|
|
|
def test_responses_protocol_relays_error_envelope(local_db, monkeypatch):
|
|
monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeResponsesClient_openai_gateway)
|
|
FakeResponsesClient_openai_gateway.behavior = "error-envelope"
|
|
_setup_responses_route_openai_gateway()
|
|
try:
|
|
svc = GatewayService()
|
|
cfg = svc.effective_config()
|
|
cfg["gateway_vision_enabled"] = False
|
|
rt = svc.runtime()
|
|
resp = run_async(
|
|
rt.handle_chat(
|
|
_chat_body_openai_gateway(), cfg, ("guest", "test"), "test", "default"
|
|
)
|
|
)
|
|
assert resp.status_code == 429
|
|
payload = json.loads(bytes(resp.body).decode())
|
|
assert payload["error"]["message"] == "slow down"
|
|
assert payload["error"]["code"] == "rate_limit_exceeded"
|
|
finally:
|
|
FakeResponsesClient_openai_gateway.behavior = "ok"
|
|
_teardown_responses_route_openai_gateway()
|
|
|
|
|
|
def test_responses_protocol_attestation_denied_logs_guidance(
|
|
local_db, monkeypatch
|
|
):
|
|
monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeResponsesClient_openai_gateway)
|
|
FakeResponsesClient_openai_gateway.behavior = "attestation-denied"
|
|
_setup_responses_route_openai_gateway()
|
|
try:
|
|
svc = GatewayService()
|
|
cfg = svc.effective_config()
|
|
cfg["gateway_vision_enabled"] = False
|
|
rt = svc.runtime()
|
|
logged = []
|
|
resp = run_async(
|
|
rt.handle_chat(
|
|
_chat_body_openai_gateway(),
|
|
cfg,
|
|
("guest", "test"),
|
|
"test",
|
|
"default",
|
|
log=logged.append,
|
|
)
|
|
)
|
|
assert resp.status_code == 403
|
|
payload = json.loads(bytes(resp.body).decode())
|
|
assert payload["error"]["code"] == "client_attestation_denied"
|
|
assert any("pay-as-you-go" in line for line in logged)
|
|
finally:
|
|
FakeResponsesClient_openai_gateway.behavior = "ok"
|
|
_teardown_responses_route_openai_gateway()
|
|
|
|
|
|
def test_upstream_auth_guidance_covers_401_and_403():
|
|
denied = FakeResp_openai_gateway(
|
|
status=403,
|
|
payload={
|
|
"error": {
|
|
"message": "subscription only",
|
|
"type": "permission_error",
|
|
"code": "client_attestation_denied",
|
|
}
|
|
},
|
|
)
|
|
guidance = gwmod._upstream_auth_guidance(denied, denied.json())
|
|
assert "pay-as-you-go" in guidance
|
|
assert "aliasing" in guidance
|
|
assert "MUSE_RESERVED_TOOL_NAMES" in guidance
|
|
other_denied = FakeResp_openai_gateway(status=403, payload={"error": "nope"})
|
|
assert (
|
|
gwmod._upstream_auth_guidance(other_denied, other_denied.json()) is None
|
|
)
|
|
unauthed = FakeResp_openai_gateway(
|
|
status=401, payload={"error": {"message": "User not found.", "code": 401}}
|
|
)
|
|
guidance = gwmod._upstream_auth_guidance(unauthed, unauthed.json())
|
|
assert "api_key" in guidance
|
|
assert "User not found." in guidance
|
|
server_error = FakeResp_openai_gateway(status=500, payload={"error": "x"})
|
|
assert (
|
|
gwmod._upstream_auth_guidance(server_error, server_error.json()) is None
|
|
)
|
|
|
|
|
|
def test_should_fallback_model_unavailable_but_not_bad_request():
|
|
unavailable = FakeResp_openai_gateway(
|
|
status=400,
|
|
payload={"error": {"message": "Model is unavailable."}},
|
|
)
|
|
assert (
|
|
gwmod._should_fallback(unavailable, None, {"circuit_open": False}) is True
|
|
)
|
|
malformed = FakeResp_openai_gateway(
|
|
status=400, payload={"error": "bad schema"}
|
|
)
|
|
assert (
|
|
gwmod._should_fallback(malformed, None, {"circuit_open": False}) is False
|
|
)
|
|
|
|
|
|
def test_responses_protocol_failed_object_is_502(local_db, monkeypatch):
|
|
monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeResponsesClient_openai_gateway)
|
|
FakeResponsesClient_openai_gateway.behavior = "failed-object"
|
|
_setup_responses_route_openai_gateway()
|
|
try:
|
|
svc = GatewayService()
|
|
cfg = svc.effective_config()
|
|
cfg["gateway_vision_enabled"] = False
|
|
rt = svc.runtime()
|
|
resp = run_async(
|
|
rt.handle_chat(
|
|
_chat_body_openai_gateway(), cfg, ("guest", "test"), "test", "default"
|
|
)
|
|
)
|
|
assert resp.status_code == 502
|
|
finally:
|
|
FakeResponsesClient_openai_gateway.behavior = "ok"
|
|
_teardown_responses_route_openai_gateway()
|
|
|
|
|
|
def test_responses_protocol_client_reasoning_effort_passes_through(
|
|
local_db, monkeypatch
|
|
):
|
|
monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeResponsesClient_openai_gateway)
|
|
FakeResponsesClient_openai_gateway.behavior = "ok"
|
|
_setup_responses_route_openai_gateway()
|
|
try:
|
|
svc = GatewayService()
|
|
cfg = svc.effective_config()
|
|
cfg["gateway_vision_enabled"] = False
|
|
rt = svc.runtime()
|
|
run_async(
|
|
rt.handle_chat(
|
|
_chat_body_openai_gateway(reasoning_effort="high"),
|
|
cfg,
|
|
("guest", "test"),
|
|
"test",
|
|
"default",
|
|
)
|
|
)
|
|
_, sent = rt._client.calls[-1]
|
|
assert sent["reasoning"] == {"effort": "high"}
|
|
finally:
|
|
_teardown_responses_route_openai_gateway()
|
|
|
|
|
|
def test_responses_protocol_default_provider(local_db, monkeypatch):
|
|
monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeResponsesClient_openai_gateway)
|
|
FakeResponsesClient_openai_gateway.behavior = "ok"
|
|
svc = GatewayService()
|
|
cfg = svc.effective_config()
|
|
cfg["gateway_vision_enabled"] = False
|
|
cfg["gateway_upstream_url"] = "https://api.meta.ai/v1/responses"
|
|
cfg["gateway_upstream_protocol"] = "responses"
|
|
cfg["gateway_model"] = "muse-spark-1.3-contributor"
|
|
rt = svc.runtime()
|
|
resp = run_async(
|
|
rt.handle_chat(
|
|
{"messages": [{"role": "user", "content": "hi"}]},
|
|
cfg,
|
|
("guest", "test"),
|
|
"test",
|
|
"default",
|
|
)
|
|
)
|
|
url, sent = rt._client.calls[-1]
|
|
assert str(url) == "https://api.meta.ai/v1/responses"
|
|
assert sent["input"] == [{"type": "message", "role": "user", "content": "hi"}]
|
|
assert resp.status_code == 200
|
|
payload = json.loads(bytes(resp.body).decode())
|
|
assert payload["object"] == "chat.completion"
|
|
|
|
|
|
EMULATED_CALL_TEXT_openai_gateway = (
|
|
"Sure.\n<tool_call>\n"
|
|
'{"name": "bash", "arguments": {"command": "ls"}}\n'
|
|
"</tool_call>"
|
|
)
|
|
BASH_TOOL_openai_gateway = {
|
|
"type": "function",
|
|
"function": {
|
|
"name": "bash",
|
|
"description": "Run a shell command",
|
|
"parameters": {
|
|
"type": "object",
|
|
"properties": {"command": {"type": "string"}},
|
|
"required": ["command"],
|
|
},
|
|
},
|
|
}
|
|
|
|
|
|
def _emulated_obj_openai_gateway(model, text):
|
|
obj = _resp_obj_openai_gateway(model)
|
|
obj["output"][0]["content"][0]["text"] = text
|
|
return obj
|
|
|
|
|
|
class FakeEmulatedClient_openai_gateway(FakeResponsesClient_openai_gateway):
|
|
text = EMULATED_CALL_TEXT_openai_gateway
|
|
|
|
async def send(self, request, stream=False):
|
|
self.calls.append((request.url, request.json_body))
|
|
model = (request.json_body or {}).get("model", "")
|
|
obj = _emulated_obj_openai_gateway(model, self.text)
|
|
events = [{"type": "response.created", "response": {"id": "resp_1"}}]
|
|
for i in range(0, len(self.text), 4):
|
|
events.append(
|
|
{"type": "response.output_text.delta", "delta": self.text[i : i + 4]}
|
|
)
|
|
events.append({"type": "response.completed", "response": obj})
|
|
return FakeResponsesResp_openai_gateway(payload=obj, events=events)
|
|
|
|
|
|
def _setup_emulated_route_openai_gateway():
|
|
from devplacepy.services.openai_gateway import routing
|
|
|
|
routing.provider_store.set(
|
|
routing.ProviderIn(
|
|
name="gwmeta",
|
|
base_url="https://api.meta.ai/v1/responses",
|
|
api_key="sk-test",
|
|
protocol="responses",
|
|
)
|
|
)
|
|
routing.model_store.set(
|
|
routing.ModelRouteIn(
|
|
source_model="meta-chat",
|
|
provider="gwmeta",
|
|
target_model="muse-spark-1.3-contributor",
|
|
kind="chat",
|
|
tools_mode="emulated",
|
|
)
|
|
)
|
|
|
|
|
|
def _emulated_chat_openai_gateway(monkeypatch, body, **tools):
|
|
monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeEmulatedClient_openai_gateway)
|
|
_setup_emulated_route_openai_gateway()
|
|
svc = GatewayService()
|
|
cfg = svc.effective_config()
|
|
cfg["gateway_vision_enabled"] = False
|
|
rt = svc.runtime()
|
|
resp = run_async(rt.handle_chat(body, cfg, ("guest", "test"), "test", "default"))
|
|
return rt, resp
|
|
|
|
|
|
def test_emulated_tools_meta_responses_nonstream(local_db, monkeypatch):
|
|
body = _chat_body_openai_gateway(tools=[BASH_TOOL_openai_gateway], tool_choice="auto")
|
|
try:
|
|
rt, resp = _emulated_chat_openai_gateway(monkeypatch, body)
|
|
url, sent = rt._client.calls[-1]
|
|
assert str(url) == "https://api.meta.ai/v1/responses"
|
|
assert "tools" not in sent and "tool_choice" not in sent
|
|
assert "<tools>" in sent["instructions"] and '"bash"' in sent["instructions"]
|
|
assert resp.status_code == 200
|
|
message = json.loads(bytes(resp.body).decode())["choices"][0]
|
|
assert message["finish_reason"] == "tool_calls"
|
|
call = message["message"]["tool_calls"][0]
|
|
assert call["function"]["name"] == "bash"
|
|
assert json.loads(call["function"]["arguments"]) == {"command": "ls"}
|
|
assert message["message"]["content"] == "Sure."
|
|
finally:
|
|
_teardown_responses_route_openai_gateway()
|
|
|
|
|
|
def test_emulated_tools_meta_responses_stream(local_db, monkeypatch):
|
|
body = _chat_body_openai_gateway(
|
|
stream=True, tools=[BASH_TOOL_openai_gateway]
|
|
)
|
|
try:
|
|
rt, resp = _emulated_chat_openai_gateway(monkeypatch, body)
|
|
|
|
async def drain():
|
|
out = []
|
|
async for chunk in resp.body_iterator:
|
|
out.append(chunk if isinstance(chunk, str) else chunk.decode())
|
|
return "".join(out)
|
|
|
|
raw = run_async(drain())
|
|
chunks = [
|
|
json.loads(line[5:])
|
|
for line in raw.splitlines()
|
|
if line.startswith("data:") and "[DONE]" not in line
|
|
]
|
|
text = "".join(
|
|
c["choices"][0]["delta"].get("content", "")
|
|
for c in chunks
|
|
if c.get("choices")
|
|
)
|
|
calls = [
|
|
tc
|
|
for c in chunks
|
|
for choice in c.get("choices") or []
|
|
for tc in choice["delta"].get("tool_calls") or []
|
|
]
|
|
assert "<tool_call>" not in text and text.strip() == "Sure."
|
|
assert len(calls) == 1 and calls[0]["function"]["name"] == "bash"
|
|
finishes = [
|
|
c["choices"][0]["finish_reason"]
|
|
for c in chunks
|
|
if c.get("choices") and c["choices"][0].get("finish_reason")
|
|
]
|
|
assert finishes[-1] == "tool_calls"
|
|
assert raw.rstrip().endswith("[DONE]")
|
|
finally:
|
|
_teardown_responses_route_openai_gateway()
|
|
|
|
|
|
def test_emulated_tools_convert_history_for_the_meta_transpiler(local_db, monkeypatch):
|
|
history = [
|
|
{"role": "user", "content": "go"},
|
|
{
|
|
"role": "assistant",
|
|
"content": None,
|
|
"tool_calls": [
|
|
{
|
|
"id": "call_1",
|
|
"type": "function",
|
|
"function": {"name": "bash", "arguments": '{"command": "ls"}'},
|
|
}
|
|
],
|
|
},
|
|
{"role": "tool", "tool_call_id": "call_1", "content": "a.txt"},
|
|
]
|
|
body = _chat_body_openai_gateway(messages=history, tools=[BASH_TOOL_openai_gateway])
|
|
try:
|
|
rt, resp = _emulated_chat_openai_gateway(monkeypatch, body)
|
|
_, sent = rt._client.calls[-1]
|
|
assert resp.status_code == 200
|
|
assert all(item.get("type") == "message" for item in sent["input"])
|
|
joined = json.dumps(sent["input"])
|
|
assert "<tool_call>" in joined and "<tool_response>" in joined and "a.txt" in joined
|
|
assert "function_call" not in joined
|
|
finally:
|
|
_teardown_responses_route_openai_gateway()
|
|
|
|
|
|
def test_emulated_tools_serve_large_catalogs_with_no_cap(local_db, monkeypatch):
|
|
tools = [
|
|
{
|
|
"type": "function",
|
|
"function": {
|
|
"name": f"tool_{index}",
|
|
"description": f"tool number {index}",
|
|
"parameters": {"type": "object", "properties": {}, "required": []},
|
|
},
|
|
}
|
|
for index in range(200)
|
|
]
|
|
body = _chat_body_openai_gateway(tools=tools, tool_choice="auto")
|
|
try:
|
|
rt, resp = _emulated_chat_openai_gateway(monkeypatch, body)
|
|
assert resp.status_code == 200
|
|
assert len(rt._client.calls) == 1
|
|
_, sent = rt._client.calls[-1]
|
|
assert "tools" not in sent and "tool_choice" not in sent
|
|
assert '"name": "tool_0"' in sent["instructions"]
|
|
assert '"name": "tool_199"' in sent["instructions"]
|
|
assert len(rt._catalog_cache._entries) == 1
|
|
finally:
|
|
_teardown_responses_route_openai_gateway()
|
|
|
|
|
|
def test_native_route_keeps_forwarding_tools(local_db, monkeypatch):
|
|
monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeResponsesClient_openai_gateway)
|
|
FakeResponsesClient_openai_gateway.behavior = "ok"
|
|
_setup_responses_route_openai_gateway()
|
|
try:
|
|
svc = GatewayService()
|
|
cfg = svc.effective_config()
|
|
cfg["gateway_vision_enabled"] = False
|
|
rt = svc.runtime()
|
|
run_async(
|
|
rt.handle_chat(
|
|
_chat_body_openai_gateway(tools=[BASH_TOOL_openai_gateway]),
|
|
cfg,
|
|
("guest", "test"),
|
|
"test",
|
|
"default",
|
|
)
|
|
)
|
|
_, sent = rt._client.calls[-1]
|
|
assert "tools" in sent
|
|
assert "<tools>" not in sent["instructions"]
|
|
finally:
|
|
_teardown_responses_route_openai_gateway()
|