Files
devplacepy/tests/unit/services/openai_gateway/gateway.py
T

4167 lines
142 KiB
Python

# retoor <retoor@molodetz.nl>
import json
from starlette.requests import Request
from tests.conftest import BASE_URL, run_async
from devplacepy.database import get_table, set_setting
from devplacepy.utils import generate_uid
import devplacepy.services.openai_gateway.gateway as gwmod
from devplacepy.services.openai_gateway import GatewayService
from devplacepy.services.openai_gateway import auth_throttle
class FakeResp_openai_gateway:
def __init__(
self,
status=200,
payload=None,
ctype="application/json",
content=b"",
extra_headers=None,
):
self.status_code = status
self._payload = payload
self.text = json.dumps(payload) if payload is not None else ""
self.headers = {"content-type": ctype, **(extra_headers or {})}
self.content = content
def json(self):
if self._payload is None:
raise ValueError("no json")
return self._payload
async def aread(self):
return self.content or self.text.encode()
async def aclose(self):
pass
async def aiter_lines(self):
payload = self._payload or {}
chunk_id = payload.get("id", "x")
model = payload.get("model", "")
try:
content = payload["choices"][0]["message"].get("content") or ""
except (KeyError, IndexError, TypeError):
content = ""
try:
reasoning = payload["choices"][0]["message"].get("reasoning") or ""
except (KeyError, IndexError, TypeError):
reasoning = ""
def frame(delta=None, finish=None, usage=None):
body = {"id": chunk_id, "object": "chat.completion.chunk", "model": model}
if usage is not None:
body["choices"] = []
body["usage"] = usage
else:
body["choices"] = [{"index": 0, "delta": delta or {}, "finish_reason": finish}]
return f"data: {json.dumps(body)}"
yield frame({"role": "assistant"})
for i in range(0, len(reasoning), 5):
yield frame({"reasoning": reasoning[i : i + 5]})
for i in range(0, len(content), 5):
yield frame({"content": content[i : i + 5]})
yield frame({}, finish="stop")
if payload.get("usage"):
yield frame(usage=payload["usage"])
yield "data: [DONE]"
class FakeRequest:
def __init__(self, method, url, json_body, headers=None):
self.method = method
self.url = url
self.json_body = json_body
self.headers = headers or {}
self.extensions = {}
class FakeClient_openai_gateway:
def __init__(self, *a, **k):
self.calls = []
def build_request(self, method, url, headers=None, json=None, content=None):
return FakeRequest(method, url, json)
async def send(self, request, stream=False):
self.calls.append((request.url, request.json_body))
body = request.json_body or {}
return FakeResp_openai_gateway(
payload={
"id": "x",
"model": body.get("model"),
"choices": [{"message": {"content": "hi there"}}],
}
)
async def post(self, url, headers=None, json=None, timeout=None):
self.calls.append((url, json))
return FakeResp_openai_gateway(
payload={
"id": "x",
"model": json.get("model"),
"choices": [{"message": {"content": "hi there"}}],
}
)
async def request(self, method, url, headers=None, content=None):
return FakeResp_openai_gateway(payload={"ok": True})
async def aclose(self):
pass
def _make_request_openai_gateway(headers=None, cookies=None):
headers = headers or {}
raw = [(k.lower().encode(), v.encode()) for k, v in headers.items()]
if cookies:
raw.append(
(b"cookie", "; ".join(f"{k}={v}" for k, v in cookies.items()).encode())
)
return Request(
{
"type": "http",
"method": "POST",
"path": "/openai/v1/chat/completions",
"query_string": b"",
"headers": raw,
"state": {},
}
)
def _make_admin_openai_gateway(role="Admin"):
username = f"gw_{generate_uid()[:8]}"
api_key = generate_uid()
get_table("users").insert(
{
"uid": generate_uid(),
"username": username,
"terms_version": "1",
"email": f"{username}@t.dev",
"api_key": api_key,
"role": role,
"is_active": True,
}
)
return username, api_key
def _config(page, **fields):
page.request.post(f"{BASE_URL}/admin/services/openai/config", form=fields)
def test_model_is_forced(local_db, monkeypatch):
monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeClient_openai_gateway)
svc = GatewayService()
cfg = svc.effective_config()
cfg["gateway_force_model"] = True
cfg["gateway_model"] = "deepseek-chat"
rt = svc.runtime()
run_async(
rt.handle_chat(
{"model": "gpt-4", "messages": [{"role": "user", "content": "hi"}]},
cfg,
("guest", "test"),
"test",
"default",
)
)
assert rt._client.calls[-1][1]["model"] == "deepseek-chat"
def test_model_route_overrides_upstream(local_db, monkeypatch):
from devplacepy.services.openai_gateway import routing
monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeClient_openai_gateway)
routing.provider_store.set(
routing.ProviderIn(
name="gwroute",
base_url="https://routed.example/v1/chat/completions",
api_key="routed-key",
)
)
routing.model_store.set(
routing.ModelRouteIn(
source_model="custom-pro",
provider="gwroute",
target_model="vendor/pro",
kind="chat",
price_output_per_m=9.0,
)
)
try:
svc = GatewayService()
cfg = svc.effective_config()
cfg["gateway_vision_enabled"] = False
rt = svc.runtime()
run_async(
rt.handle_chat(
{"model": "custom-pro", "messages": [{"role": "user", "content": "hi"}]},
cfg,
("guest", "test"),
"test",
"default",
)
)
url, body = rt._client.calls[-1]
assert str(url) == "https://routed.example/v1/chat/completions"
assert body["model"] == "vendor/pro"
finally:
routing.model_store.remove("custom-pro")
routing.provider_store.remove("gwroute")
def test_vision_rewrites_image_to_text(local_db, monkeypatch):
monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeClient_openai_gateway)
svc = GatewayService()
cfg = svc.effective_config()
cfg["gateway_vision_enabled"] = True
cfg["gateway_vision_key"] = "" # no key -> placeholder text, still rewrites to str
rt = svc.runtime()
msgs = [
{
"role": "user",
"content": [
{"type": "text", "text": "what is this"},
{"type": "image_url", "image_url": {"url": "http://x/y.png"}},
],
}
]
run_async(rt.handle_chat({"messages": msgs}, cfg, ("guest", "test"), "test", "default"))
sent = rt._client.calls[-1][1]["messages"][0]["content"]
assert isinstance(sent, str) and "vision" in sent.lower()
def test_streaming_emits_sse(local_db, monkeypatch):
monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeClient_openai_gateway)
svc = GatewayService()
cfg = svc.effective_config()
rt = svc.runtime()
resp = run_async(
rt.handle_chat(
{"messages": [{"role": "user", "content": "hi"}], "stream": True},
cfg,
("guest", "test"),
"test",
"default",
)
)
async def drain():
out = []
async for chunk in resp.body_iterator:
out.append(chunk if isinstance(chunk, str) else chunk.decode())
return "".join(out)
body = run_async(drain())
assert "chat.completion.chunk" in body
assert "[DONE]" in body
def test_authorize_static_access_key(local_db):
set_setting("gateway_require_auth", "1")
set_setting("gateway_access_key", "topsecret")
try:
svc = GatewayService()
assert svc.authorize(_make_request_openai_gateway(headers={"X-API-KEY": "topsecret"})) is True
assert svc.authorize(_make_request_openai_gateway(headers={"X-API-KEY": "wrong"})) is False
finally:
set_setting("gateway_access_key", "")
set_setting("gateway_require_auth", "1")
auth_throttle.clear()
def test_authorize_admin_and_user_toggles(local_db):
set_setting("gateway_require_auth", "1")
set_setting("gateway_access_key", "")
set_setting("gateway_allow_admins", "1")
set_setting("gateway_allow_users", "0")
try:
_, admin_key = _make_admin_openai_gateway(role="Admin")
_, member_key = _make_admin_openai_gateway(role="Member")
svc = GatewayService()
assert (
svc.authorize(_make_request_openai_gateway(headers={"Authorization": f"Bearer {admin_key}"}))
is True
)
assert svc.authorize(_make_request_openai_gateway(headers={"X-API-KEY": member_key})) is False
set_setting("gateway_allow_users", "1")
assert svc.authorize(_make_request_openai_gateway(headers={"X-API-KEY": member_key})) is True
finally:
set_setting("gateway_allow_admins", "1")
set_setting("gateway_allow_users", "1")
set_setting("gateway_require_auth", "1")
set_setting("gateway_access_key", "")
auth_throttle.clear()
def test_authorize_require_auth_off_is_open(local_db):
set_setting("gateway_require_auth", "0")
try:
svc = GatewayService()
assert svc.authorize(_make_request_openai_gateway()) is True
finally:
set_setting("gateway_require_auth", "1")
def test_authorize_throttles_repeated_failed_auth_from_one_ip(local_db):
import pytest
from fastapi import HTTPException
set_setting("gateway_require_auth", "1")
set_setting("gateway_access_key", "")
set_setting("gateway_auth_throttle_max_failures", "3")
set_setting("gateway_auth_throttle_window_seconds", "60")
auth_throttle.clear()
try:
svc = GatewayService()
ip = "203.0.113.10"
for _ in range(3):
assert (
svc.authorize(_make_request_openai_gateway(headers={"X-Real-IP": ip}))
is False
)
with pytest.raises(HTTPException) as excinfo:
svc.authorize(_make_request_openai_gateway(headers={"X-Real-IP": ip}))
assert excinfo.value.status_code == 429
assert "Retry-After" in excinfo.value.headers
finally:
set_setting("gateway_auth_throttle_max_failures", "10")
set_setting("gateway_auth_throttle_window_seconds", "60")
set_setting("gateway_require_auth", "1")
auth_throttle.clear()
def test_authorize_throttle_does_not_affect_other_ips(local_db):
set_setting("gateway_require_auth", "1")
set_setting("gateway_access_key", "")
set_setting("gateway_auth_throttle_max_failures", "3")
set_setting("gateway_auth_throttle_window_seconds", "60")
auth_throttle.clear()
try:
svc = GatewayService()
throttled_ip = "203.0.113.20"
other_ip = "203.0.113.21"
for _ in range(3):
svc.authorize(_make_request_openai_gateway(headers={"X-Real-IP": throttled_ip}))
assert auth_throttle.is_throttled(throttled_ip, 3, 60) is True
assert (
svc.authorize(_make_request_openai_gateway(headers={"X-Real-IP": other_ip}))
is False
)
assert auth_throttle.is_throttled(other_ip, 3, 60) is False
finally:
set_setting("gateway_auth_throttle_max_failures", "10")
set_setting("gateway_auth_throttle_window_seconds", "60")
set_setting("gateway_require_auth", "1")
auth_throttle.clear()
def test_authorize_valid_credentials_succeed_from_throttled_ip(local_db):
set_setting("gateway_require_auth", "1")
set_setting("gateway_access_key", "")
set_setting("gateway_allow_users", "1")
set_setting("gateway_auth_throttle_max_failures", "3")
set_setting("gateway_auth_throttle_window_seconds", "60")
auth_throttle.clear()
try:
svc = GatewayService()
ip = "203.0.113.30"
_, member_key = _make_admin_openai_gateway(role="Member")
for _ in range(3):
svc.authorize(_make_request_openai_gateway(headers={"X-Real-IP": ip}))
assert auth_throttle.is_throttled(ip, 3, 60) is True
assert (
svc.authorize(
_make_request_openai_gateway(
headers={"X-Real-IP": ip, "X-API-KEY": member_key}
)
)
is True
)
set_setting("gateway_access_key", "topsecret")
assert (
svc.authorize(
_make_request_openai_gateway(
headers={"X-Real-IP": ip, "X-API-KEY": "topsecret"}
)
)
is True
)
finally:
set_setting("gateway_access_key", "")
set_setting("gateway_auth_throttle_max_failures", "10")
set_setting("gateway_auth_throttle_window_seconds", "60")
set_setting("gateway_require_auth", "1")
auth_throttle.clear()
def test_authorize_throttle_resets_after_window(local_db, monkeypatch):
set_setting("gateway_require_auth", "1")
set_setting("gateway_access_key", "")
set_setting("gateway_auth_throttle_max_failures", "3")
set_setting("gateway_auth_throttle_window_seconds", "60")
auth_throttle.clear()
try:
now = [2000.0]
monkeypatch.setattr(auth_throttle.time, "time", lambda: now[0])
svc = GatewayService()
ip = "203.0.113.40"
for _ in range(3):
svc.authorize(_make_request_openai_gateway(headers={"X-Real-IP": ip}))
assert auth_throttle.is_throttled(ip, 3, 60) is True
now[0] += 61
assert (
svc.authorize(_make_request_openai_gateway(headers={"X-Real-IP": ip}))
is False
)
finally:
set_setting("gateway_auth_throttle_max_failures", "10")
set_setting("gateway_auth_throttle_window_seconds", "60")
set_setting("gateway_require_auth", "1")
auth_throttle.clear()
class FakeEmbedClient_openai_gateway:
def __init__(self, *a, **k):
self.calls = []
def build_request(self, method, url, headers=None, json=None, content=None):
return FakeRequest(method, url, json)
async def send(self, request, stream=False):
self.calls.append((request.url, request.json_body))
body = request.json_body or {}
return FakeResp_openai_gateway(
payload={
"object": "list",
"model": body.get("model"),
"data": [{"object": "embedding", "index": 0, "embedding": [0.1, 0.2]}],
"usage": {"prompt_tokens": 5, "total_tokens": 5},
}
)
async def aclose(self):
pass
def test_embeddings_remaps_alias_to_configured_model(local_db, monkeypatch):
monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeEmbedClient_openai_gateway)
monkeypatch.setenv("OPENROUTER_API_KEY", "test-embed-key")
svc = GatewayService()
cfg = svc.effective_config()
cfg["gateway_force_model"] = False
cfg["gateway_embed_enabled"] = True
cfg["gateway_embed_model"] = "qwen/qwen3-embedding-8b"
rt = svc.runtime()
run_async(
rt.handle_embeddings(
{"model": "molodetz~embed", "input": "hello"},
cfg,
("guest", "test"),
"test",
"default",
)
)
assert rt._client.calls[-1][1]["model"] == "qwen/qwen3-embedding-8b"
def test_embeddings_success_records_ledger(local_db, monkeypatch):
monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeEmbedClient_openai_gateway)
monkeypatch.setenv("OPENROUTER_API_KEY", "test-embed-key")
svc = GatewayService()
cfg = svc.effective_config()
cfg["gateway_force_model"] = True
cfg["gateway_embed_enabled"] = True
cfg["gateway_embed_model"] = "qwen/qwen3-embedding-8b"
rt = svc.runtime()
before = rt.embed_calls
resp = run_async(
rt.handle_embeddings(
{"model": "molodetz~embed", "input": "hello"},
cfg,
("guest", "ledger_probe"),
"test",
"default",
)
)
assert resp.status_code == 200
payload = json.loads(bytes(resp.body).decode())
assert payload["data"][0]["embedding"] == [0.1, 0.2]
assert rt.embed_calls == before + 1
row = get_table("gateway_usage_ledger").find_one(owner_id="ledger_probe")
assert row is not None
assert row["backend"] == "embed"
assert row["endpoint"] == "embeddings"
assert row["requested_model"] == "molodetz~embed"
assert row["model"] == "qwen/qwen3-embedding-8b"
assert row["success"] == 1
def test_embeddings_disabled_returns_503(local_db, monkeypatch):
monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeEmbedClient_openai_gateway)
svc = GatewayService()
cfg = svc.effective_config()
cfg["gateway_embed_enabled"] = False
rt = svc.runtime()
resp = run_async(
rt.handle_embeddings(
{"input": "hello"}, cfg, ("guest", "test"), "test", "default"
)
)
assert resp.status_code == 503
def test_embeddings_missing_key_returns_clean_error_not_unauthenticated_call(
local_db, monkeypatch
):
monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeEmbedClient_openai_gateway)
monkeypatch.delenv("OPENROUTER_API_KEY", raising=False)
monkeypatch.delenv("DEEPSEEK_API_KEY", raising=False)
set_setting("gateway_api_key", "")
set_setting("gateway_vision_key", "")
set_setting("gateway_embed_key", "")
try:
svc = GatewayService()
cfg = svc.effective_config()
assert cfg["gateway_embed_key"] == ""
cfg["gateway_embed_enabled"] = True
rt = svc.runtime()
resp = run_async(
rt.handle_embeddings(
{"input": "hello"}, cfg, ("guest", "no_key_probe"), "test", "default"
)
)
assert resp.status_code == 503
payload = json.loads(bytes(resp.body).decode())
assert payload["error"]["type"] == "embeddings_not_configured"
assert rt._client is None
finally:
set_setting("gateway_api_key", "")
set_setting("gateway_vision_key", "")
set_setting("gateway_embed_key", "")
class FakeImageClient_openai_gateway:
def __init__(self, *a, **k):
self.calls = []
def build_request(self, method, url, headers=None, json=None, content=None):
return FakeRequest(method, url, json)
async def send(self, request, stream=False):
self.calls.append((request.url, request.json_body))
body = request.json_body or {}
return FakeResp_openai_gateway(
payload={
"created": 1,
"model": body.get("model"),
"data": [{"b64_json": "aGVsbG8="}],
"usage": {"cost": 0.05},
}
)
async def aclose(self):
pass
def test_images_remaps_alias_to_configured_model(local_db, monkeypatch):
monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeImageClient_openai_gateway)
svc = GatewayService()
cfg = svc.effective_config()
cfg["gateway_force_model"] = False
cfg["gateway_image_enabled"] = True
cfg["gateway_image_model"] = "black-forest-labs/flux.2-pro"
rt = svc.runtime()
run_async(
rt.handle_images(
{
"model": "molodetz-img-small",
"prompt": "award emblem",
"size": "512x512",
},
cfg,
("guest", "test"),
"test",
"default",
)
)
assert rt._client.calls[-1][1]["model"] == "black-forest-labs/flux.2-pro"
def test_image_route_overrides_upstream(local_db, monkeypatch):
from devplacepy.services.openai_gateway import routing
monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeImageClient_openai_gateway)
routing.provider_store.set(
routing.ProviderIn(
name="gwimg",
base_url="https://routed.example/v1/chat/completions",
api_key="img-key",
)
)
routing.model_store.set(
routing.ModelRouteIn(
source_model="custom-img",
provider="gwimg",
target_model="vendor/flux-pro",
kind="image",
price_input_per_m=0.06,
)
)
try:
svc = GatewayService()
cfg = svc.effective_config()
rt = svc.runtime()
run_async(
rt.handle_images(
{
"model": "custom-img",
"prompt": "trophy",
"response_format": "b64_json",
},
cfg,
("guest", "test"),
"test",
"default",
)
)
url, body = rt._client.calls[-1]
assert str(url) == "https://routed.example/v1/images"
assert body["model"] == "vendor/flux-pro"
finally:
routing.model_store.remove("custom-img")
routing.provider_store.remove("gwimg")
def test_images_success_records_ledger(local_db, monkeypatch):
monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeImageClient_openai_gateway)
svc = GatewayService()
cfg = svc.effective_config()
cfg["gateway_image_enabled"] = True
cfg["gateway_image_model"] = "black-forest-labs/flux.2-pro"
rt = svc.runtime()
before = rt.image_calls
resp = run_async(
rt.handle_images(
{"model": "molodetz-img-small", "prompt": "badge"},
cfg,
("guest", "img_ledger"),
"test",
"default",
)
)
assert resp.status_code == 200
payload = json.loads(bytes(resp.body).decode())
assert payload["data"][0]["b64_json"] == "aGVsbG8="
assert rt.image_calls == before + 1
row = get_table("gateway_usage_ledger").find_one(owner_id="img_ledger")
assert row is not None
assert row["backend"] == "image"
assert row["endpoint"] == "images/generations"
assert row["requested_model"] == "molodetz-img-small"
assert row["model"] == "black-forest-labs/flux.2-pro"
assert row["success"] == 1
assert float(row["cost_usd"]) == 0.05
def test_images_disabled_returns_503(local_db, monkeypatch):
monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeImageClient_openai_gateway)
svc = GatewayService()
cfg = svc.effective_config()
cfg["gateway_image_enabled"] = False
rt = svc.runtime()
resp = run_async(
rt.handle_images(
{"prompt": "badge"}, cfg, ("guest", "test"), "test", "default"
)
)
assert resp.status_code == 503
def test_compute_cost_embed_branch():
from devplacepy.services.openai_gateway.usage import (
Pricing,
compute_cost,
normalize_usage,
)
pricing = Pricing(
chat_cache_hit_per_m=0.0,
chat_cache_miss_per_m=0.0,
chat_output_per_m=0.0,
vision_input_per_m=0.0,
vision_output_per_m=0.0,
embed_input_per_m=0.01,
)
usage = {"prompt_tokens": 1_000_000, "total_tokens": 1_000_000}
norm = normalize_usage(usage)
total, input_cost, output_cost, native = compute_cost(
usage, norm, pricing, "embed"
)
assert native is False
assert output_cost == 0.0
assert abs(total - 0.01) < 1e-9
assert abs(input_cost - 0.01) < 1e-9
def test_app_reference_stored_in_ledger_for_chat(local_db, monkeypatch):
monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeClient_openai_gateway)
svc = GatewayService()
cfg = svc.effective_config()
cfg["gateway_force_model"] = True
cfg["gateway_model"] = "deepseek-chat"
rt = svc.runtime()
run_async(
rt.handle_chat(
{"messages": [{"role": "user", "content": "ref_test"}]},
cfg,
("guest", "ref_probe"),
"test-ua",
"my-custom-app",
)
)
row = get_table("gateway_usage_ledger").find_one(owner_id="ref_probe")
assert row is not None
assert row["app_reference"] == "my-custom-app"
assert row["backend"] == "chat"
def test_app_reference_defaults_when_not_passed(local_db, monkeypatch):
monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeClient_openai_gateway)
svc = GatewayService()
cfg = svc.effective_config()
cfg["gateway_force_model"] = True
cfg["gateway_model"] = "deepseek-chat"
rt = svc.runtime()
run_async(
rt.handle_chat(
{"messages": [{"role": "user", "content": "default_test"}]},
cfg,
("guest", "default_probe"),
"test-ua",
"default",
)
)
row = get_table("gateway_usage_ledger").find_one(owner_id="default_probe")
assert row is not None
assert row["app_reference"] == "default"
def test_app_reference_stored_in_ledger_for_embeddings(local_db, monkeypatch):
monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeEmbedClient_openai_gateway)
monkeypatch.setenv("OPENROUTER_API_KEY", "test-embed-key")
svc = GatewayService()
cfg = svc.effective_config()
cfg["gateway_embed_enabled"] = True
cfg["gateway_embed_model"] = "qwen/qwen3-embedding-8b"
rt = svc.runtime()
run_async(
rt.handle_embeddings(
{"input": "hello"},
cfg,
("guest", "embed_ref"),
"test-ua",
"devplace-embed-test-v-1-0-0",
)
)
row = get_table("gateway_usage_ledger").find_one(owner_id="embed_ref")
assert row is not None
assert row["app_reference"] == "devplace-embed-test-v-1-0-0"
assert row["backend"] == "embed"
def test_app_reference_stored_in_ledger_for_images(local_db, monkeypatch):
monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeImageClient_openai_gateway)
svc = GatewayService()
cfg = svc.effective_config()
cfg["gateway_image_enabled"] = True
cfg["gateway_image_model"] = "black-forest-labs/flux.2-pro"
rt = svc.runtime()
run_async(
rt.handle_images(
{"prompt": "ref badge"},
cfg,
("guest", "img_ref"),
"test-ua",
"devplace-image-test-v-1-0-0",
)
)
row = get_table("gateway_usage_ledger").find_one(owner_id="img_ref")
assert row is not None
assert row["app_reference"] == "devplace-image-test-v-1-0-0"
assert row["backend"] == "image"
def test_app_reference_column_exists(local_db):
db = get_table("gateway_usage_ledger").db
if "gateway_usage_ledger" not in db.tables:
return
rows = list(db.query("PRAGMA table_info('gateway_usage_ledger')"))
column_names = [r["name"] for r in rows]
assert "app_reference" in column_names
def test_chat_unknown_model_falls_back_to_default(local_db, monkeypatch):
monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeClient_openai_gateway)
svc = GatewayService()
cfg = svc.effective_config()
cfg["gateway_force_model"] = False
cfg["gateway_model"] = "deepseek-v4-flash"
rt = svc.runtime()
run_async(
rt.handle_chat(
{"model": "nonexistent-model-v99", "messages": [{"role": "user", "content": "hi"}]},
cfg,
("guest", "fallback_test_chat"),
"test",
"default",
)
)
assert rt._client.calls[-1][1]["model"] == "deepseek-v4-flash"
class FakeFallbackClient_openai_gateway:
def __init__(self, *a, **k):
self.calls = []
def build_request(self, method, url, headers=None, json=None, content=None):
return FakeRequest(method, url, json)
async def send(self, request, stream=False):
self.calls.append((request.url, request.json_body))
body = request.json_body or {}
model = body.get("model")
if model == "primary-target":
return FakeResp_openai_gateway(status=500, payload={"error": "boom"})
return FakeResp_openai_gateway(
payload={
"id": "x",
"model": model,
"choices": [{"message": {"content": "fallback ok"}}],
}
)
async def aclose(self):
pass
class FakeAlwaysFailClient_openai_gateway:
def __init__(self, *a, **k):
self.calls = []
def build_request(self, method, url, headers=None, json=None, content=None):
return FakeRequest(method, url, json)
async def send(self, request, stream=False):
self.calls.append((request.url, request.json_body))
return FakeResp_openai_gateway(status=500, payload={"error": "boom"})
async def aclose(self):
pass
class FakeBadRequestClient_openai_gateway:
def __init__(self, *a, **k):
self.calls = []
def build_request(self, method, url, headers=None, json=None, content=None):
return FakeRequest(method, url, json)
async def send(self, request, stream=False):
self.calls.append((request.url, request.json_body))
return FakeResp_openai_gateway(
status=400, payload={"error": {"message": "invalid request: bad param"}}
)
async def aclose(self):
pass
def test_chat_does_not_fall_back_on_an_unrecoverable_bad_request(local_db, monkeypatch):
from devplacepy.services.openai_gateway import routing
monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeBadRequestClient_openai_gateway)
routing.model_store.set(
routing.ModelRouteIn(source_model="fb-badreq-backup", target_model="backup-target")
)
routing.model_store.set(
routing.ModelRouteIn(
source_model="fb-badreq-primary",
target_model="primary-target",
fallback_model="fb-badreq-backup",
)
)
try:
svc = GatewayService()
cfg = svc.effective_config()
cfg["gateway_vision_enabled"] = False
cfg["gateway_max_retries"] = 0
rt = svc.runtime()
response = run_async(
rt.handle_chat(
{"model": "fb-badreq-primary", "messages": [{"role": "user", "content": "hi"}]},
cfg,
("guest", "no_fallback_on_bad_request"),
"test",
"default",
)
)
assert response.status_code == 400
assert len(rt._client.calls) == 1
finally:
routing.model_store.remove("fb-badreq-primary")
routing.model_store.remove("fb-badreq-backup")
def test_chat_falls_back_when_the_primary_model_fails(local_db, monkeypatch):
from devplacepy.services.openai_gateway import routing
monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeFallbackClient_openai_gateway)
routing.model_store.set(
routing.ModelRouteIn(source_model="fb-backup-route", target_model="backup-target")
)
routing.model_store.set(
routing.ModelRouteIn(
source_model="fb-primary-route",
target_model="primary-target",
fallback_model="fb-backup-route",
)
)
try:
svc = GatewayService()
cfg = svc.effective_config()
cfg["gateway_vision_enabled"] = False
cfg["gateway_max_retries"] = 0
rt = svc.runtime()
response = run_async(
rt.handle_chat(
{"model": "fb-primary-route", "messages": [{"role": "user", "content": "hi"}]},
cfg,
("guest", "fallback_chat_success"),
"test",
"default",
)
)
assert response.status_code == 200
assert rt._client.calls[0][1]["model"] == "primary-target"
assert rt._client.calls[-1][1]["model"] == "backup-target"
row = get_table("gateway_usage_ledger").find_one(owner_id="fallback_chat_success")
assert row is not None
assert row["requested_model"] == "fb-primary-route"
assert row["model"] == "backup-target"
assert row["success"] == 1
finally:
routing.model_store.remove("fb-primary-route")
routing.model_store.remove("fb-backup-route")
class FakeChainClient_openai_gateway:
def __init__(self, *a, **k):
self.calls = []
def build_request(self, method, url, headers=None, json=None, content=None):
return FakeRequest(method, url, json)
async def send(self, request, stream=False):
self.calls.append((request.url, request.json_body))
body = request.json_body or {}
model = body.get("model")
if model in ("chain-a-target", "chain-b-target"):
return FakeResp_openai_gateway(status=500, payload={"error": "boom"})
return FakeResp_openai_gateway(
payload={
"id": "x",
"model": model,
"choices": [{"message": {"content": "chain ok"}}],
}
)
async def aclose(self):
pass
def test_chat_walks_fallback_chain_past_failures(local_db, monkeypatch):
from devplacepy.services.openai_gateway import routing
monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeChainClient_openai_gateway)
routing.model_store.set(
routing.ModelRouteIn(source_model="chain-c", target_model="chain-c-target")
)
routing.model_store.set(
routing.ModelRouteIn(
source_model="chain-b",
target_model="chain-b-target",
fallback_model="chain-c",
)
)
routing.model_store.set(
routing.ModelRouteIn(
source_model="chain-a",
target_model="chain-a-target",
fallback_model="chain-b",
)
)
try:
svc = GatewayService()
cfg = svc.effective_config()
cfg["gateway_vision_enabled"] = False
cfg["gateway_max_retries"] = 0
rt = svc.runtime()
response = run_async(
rt.handle_chat(
{"model": "chain-a", "messages": [{"role": "user", "content": "hi"}]},
cfg,
("guest", "chain_walk"),
"test",
"default",
)
)
assert response.status_code == 200
assert [c[1]["model"] for c in rt._client.calls] == [
"chain-a-target",
"chain-b-target",
"chain-c-target",
]
row = get_table("gateway_usage_ledger").find_one(owner_id="chain_walk")
assert row["model"] == "chain-c-target"
assert row["fallback_used_route"] == "chain-c"
finally:
routing.model_store.remove("chain-a")
routing.model_store.remove("chain-b")
routing.model_store.remove("chain-c")
def test_chat_preroutes_around_unhealthy_models(local_db, monkeypatch):
from devplacepy.services.openai_gateway import health, routing
monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeClient_openai_gateway)
routing.model_store.set(
routing.ModelRouteIn(source_model="pre-good", target_model="good-target")
)
routing.model_store.set(
routing.ModelRouteIn(
source_model="pre-bad",
target_model="bad-target",
fallback_model="pre-good",
)
)
health.record_health(
"pre-bad", health.ProbeResult(False, "http_500", "boom", 1.0)
)
try:
svc = GatewayService()
cfg = svc.effective_config()
cfg["gateway_vision_enabled"] = False
rt = svc.runtime()
logged = []
response = run_async(
rt.handle_chat(
{"model": "pre-bad", "messages": [{"role": "user", "content": "hi"}]},
cfg,
("guest", "preroute"),
"test",
"default",
log=logged.append,
)
)
assert response.status_code == 200
assert len(rt._client.calls) == 1
assert rt._client.calls[0][1]["model"] == "good-target"
assert any("is unhealthy, starting on" in line for line in logged)
row = get_table("gateway_usage_ledger").find_one(owner_id="preroute")
assert row["requested_model"] == "pre-bad"
assert row["model"] == "good-target"
finally:
routing.model_store.remove("pre-bad")
routing.model_store.remove("pre-good")
def test_chat_tries_original_when_chain_is_exhausted(local_db, monkeypatch):
from devplacepy.services.openai_gateway import health, routing
monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeClient_openai_gateway)
routing.model_store.set(
routing.ModelRouteIn(
source_model="exh-fb",
target_model="exh-fb-target",
fallback_model="exh-primary",
)
)
routing.model_store.set(
routing.ModelRouteIn(
source_model="exh-primary",
target_model="exh-primary-target",
fallback_model="exh-fb",
)
)
health.record_health(
"exh-primary", health.ProbeResult(False, "http_500", "boom", 1.0)
)
health.record_health(
"exh-fb", health.ProbeResult(False, "http_500", "boom", 1.0)
)
try:
svc = GatewayService()
cfg = svc.effective_config()
cfg["gateway_vision_enabled"] = False
rt = svc.runtime()
response = run_async(
rt.handle_chat(
{"model": "exh-primary", "messages": [{"role": "user", "content": "hi"}]},
cfg,
("guest", "exhausted"),
"test",
"default",
)
)
assert response.status_code == 200
assert [c[1]["model"] for c in rt._client.calls] == ["exh-primary-target"]
finally:
routing.model_store.remove("exh-primary")
routing.model_store.remove("exh-fb")
def test_models_endpoint_hides_unhealthy_routes(local_db):
from devplacepy.services.openai_gateway import health, routing
routing.model_store.set(
routing.ModelRouteIn(source_model="hide-me", target_model="v/hide")
)
health.record_health(
"hide-me", health.ProbeResult(False, "http_500", "boom", 1.0)
)
try:
svc = GatewayService()
payload = json.loads(svc._models_response().body.decode())
assert "hide-me" not in {m["id"] for m in payload["data"]}
health.record_health(
"hide-me", health.ProbeResult(True, "ok", "", 1.0)
)
payload = json.loads(svc._models_response().body.decode())
assert "hide-me" in {m["id"] for m in payload["data"]}
finally:
routing.model_store.remove("hide-me")
class FakeContextLengthClient_openai_gateway:
def __init__(self, *a, **k):
self.calls = []
def build_request(self, method, url, headers=None, json=None, content=None):
return FakeRequest(method, url, json)
async def send(self, request, stream=False):
self.calls.append((request.url, request.json_body))
body = request.json_body or {}
model = body.get("model")
if model == "ctx-primary-target":
return FakeResp_openai_gateway(
status=400,
payload={"error": {"message": "maximum context length exceeded"}},
)
return FakeResp_openai_gateway(
payload={
"id": "x",
"model": model,
"choices": [{"message": {"content": "fallback ok"}}],
}
)
async def aclose(self):
pass
def test_chat_falls_back_on_a_context_length_error(local_db, monkeypatch):
from devplacepy.services.openai_gateway import routing
monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeContextLengthClient_openai_gateway)
routing.model_store.set(
routing.ModelRouteIn(source_model="ctx-backup-route", target_model="ctx-backup-target")
)
routing.model_store.set(
routing.ModelRouteIn(
source_model="ctx-primary-route",
target_model="ctx-primary-target",
fallback_model="ctx-backup-route",
)
)
try:
svc = GatewayService()
cfg = svc.effective_config()
cfg["gateway_vision_enabled"] = False
cfg["gateway_max_retries"] = 0
rt = svc.runtime()
response = run_async(
rt.handle_chat(
{"model": "ctx-primary-route", "messages": [{"role": "user", "content": "hi"}]},
cfg,
("guest", "context_length_fallback_success"),
"test",
"default",
)
)
assert response.status_code == 200
assert rt._client.calls[0][1]["model"] == "ctx-primary-target"
assert rt._client.calls[-1][1]["model"] == "ctx-backup-target"
row = get_table("gateway_usage_ledger").find_one(
owner_id="context_length_fallback_success"
)
assert row["success"] == 1
assert row["fallback_used_route"] == "ctx-backup-route"
finally:
routing.model_store.remove("ctx-primary-route")
routing.model_store.remove("ctx-backup-route")
def test_chat_skips_a_too_small_primary_and_goes_straight_to_fallback(local_db, monkeypatch):
from devplacepy.services.openai_gateway import routing
monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeContextLengthClient_openai_gateway)
routing.model_store.set(
routing.ModelRouteIn(
source_model="ctx-precheck-backup",
target_model="ctx-backup-target",
context_window=1_000_000,
)
)
routing.model_store.set(
routing.ModelRouteIn(
source_model="ctx-precheck-primary",
target_model="ctx-primary-target",
context_window=50,
fallback_model="ctx-precheck-backup",
)
)
try:
svc = GatewayService()
cfg = svc.effective_config()
cfg["gateway_vision_enabled"] = False
cfg["gateway_max_retries"] = 0
rt = svc.runtime()
long_message = "word " * 2000
response = run_async(
rt.handle_chat(
{
"model": "ctx-precheck-primary",
"messages": [{"role": "user", "content": long_message}],
},
cfg,
("guest", "context_precheck_skips_primary"),
"test",
"default",
)
)
assert response.status_code == 200
# Only the fallback was ever called - the primary was known too small.
assert len(rt._client.calls) == 1
assert rt._client.calls[0][1]["model"] == "ctx-backup-target"
row = get_table("gateway_usage_ledger").find_one(
owner_id="context_precheck_skips_primary"
)
assert row["fallback_used_route"] == "ctx-precheck-backup"
finally:
routing.model_store.remove("ctx-precheck-primary")
routing.model_store.remove("ctx-precheck-backup")
def test_chat_falls_back_via_molodetz_when_the_client_sends_an_unrouted_model_name(
local_db, monkeypatch
):
from devplacepy.services.openai_gateway import routing
original = routing.model_store.get("molodetz")
routing.model_store.set(
routing.ModelRouteIn(source_model="fb-molodetz-backup", target_model="backup-target")
)
routing.model_store.set(
routing.ModelRouteIn(
source_model="molodetz",
target_model="primary-target",
fallback_model="fb-molodetz-backup",
)
)
try:
svc = GatewayService()
cfg = svc.effective_config()
cfg["gateway_vision_enabled"] = False
cfg["gateway_max_retries"] = 0
cfg["gateway_force_model"] = False
base_default_model = cfg["gateway_model"]
class FakeUnroutedPrimaryFailsClient:
def __init__(self, *a, **k):
self.calls = []
def build_request(self, method, url, headers=None, json=None, content=None):
return FakeRequest(method, url, json)
async def send(self, request, stream=False):
self.calls.append((request.url, request.json_body))
body = request.json_body or {}
model = body.get("model")
if model == base_default_model:
return FakeResp_openai_gateway(
status=402, payload={"error": {"message": "Insufficient Balance"}}
)
return FakeResp_openai_gateway(
payload={
"id": "x",
"model": model,
"choices": [{"message": {"content": "fallback ok"}}],
}
)
async def aclose(self):
pass
monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeUnroutedPrimaryFailsClient)
rt = svc.runtime()
# The client sends a model name that matches no configured route at all
# (as a caller like dpc does, which is never told the exact "molodetz"
# routing alias) - the primary call falls through to the flat/global
# default model (no route overlay applies), but a failure there must
# still consult molodetz's own configured fallback, not silently skip
# it just because the client's literal string never matched anything.
response = run_async(
rt.handle_chat(
{
"model": "some-client-side-model-name",
"messages": [{"role": "user", "content": "hi"}],
},
cfg,
("guest", "fallback_chat_unrouted_client_model"),
"test",
"default",
)
)
assert response.status_code == 200
assert rt._client.calls[0][1]["model"] == base_default_model
assert rt._client.calls[-1][1]["model"] == "backup-target"
row = get_table("gateway_usage_ledger").find_one(
owner_id="fallback_chat_unrouted_client_model"
)
assert row is not None
assert row["requested_model"] == "some-client-side-model-name"
assert row["model"] == "backup-target"
assert row["success"] == 1
finally:
routing.model_store.remove("fb-molodetz-backup")
if original is not None:
routing.model_store.set(
routing.ModelRouteIn(
**{
field: getattr(original, field)
for field in routing.ModelRouteIn.model_fields
}
)
)
else:
routing.model_store.remove("molodetz")
def test_chat_returns_the_fallback_failure_when_both_models_fail(local_db, monkeypatch):
from devplacepy.services.openai_gateway import routing
monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeAlwaysFailClient_openai_gateway)
routing.model_store.set(
routing.ModelRouteIn(source_model="fb-backup-route2", target_model="backup-target2")
)
routing.model_store.set(
routing.ModelRouteIn(
source_model="fb-primary-route2",
target_model="primary-target2",
fallback_model="fb-backup-route2",
)
)
try:
svc = GatewayService()
cfg = svc.effective_config()
cfg["gateway_vision_enabled"] = False
cfg["gateway_max_retries"] = 0
rt = svc.runtime()
response = run_async(
rt.handle_chat(
{"model": "fb-primary-route2", "messages": [{"role": "user", "content": "hi"}]},
cfg,
("guest", "fallback_chat_both_fail"),
"test",
"default",
)
)
assert response.status_code == 500
assert len(rt._client.calls) == 2
assert rt._client.calls[0][1]["model"] == "primary-target2"
assert rt._client.calls[1][1]["model"] == "backup-target2"
finally:
routing.model_store.remove("fb-primary-route2")
routing.model_store.remove("fb-backup-route2")
def test_chat_without_a_fallback_never_makes_a_second_call(local_db, monkeypatch):
from devplacepy.services.openai_gateway import routing
monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeAlwaysFailClient_openai_gateway)
routing.model_store.set(
routing.ModelRouteIn(source_model="fb-no-fallback", target_model="no-fallback-target")
)
try:
svc = GatewayService()
cfg = svc.effective_config()
cfg["gateway_vision_enabled"] = False
cfg["gateway_max_retries"] = 0
rt = svc.runtime()
run_async(
rt.handle_chat(
{"model": "fb-no-fallback", "messages": [{"role": "user", "content": "hi"}]},
cfg,
("guest", "no_fallback_configured"),
"test",
"default",
)
)
assert len(rt._client.calls) == 1
finally:
routing.model_store.remove("fb-no-fallback")
def test_embeddings_unknown_model_falls_back_to_default(local_db, monkeypatch):
monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeEmbedClient_openai_gateway)
monkeypatch.setenv("OPENROUTER_API_KEY", "test-embed-key")
svc = GatewayService()
cfg = svc.effective_config()
cfg["gateway_force_model"] = False
cfg["gateway_embed_enabled"] = True
cfg["gateway_embed_model"] = "qwen/qwen3-embedding-8b"
rt = svc.runtime()
run_async(
rt.handle_embeddings(
{"model": "nonexistent-embed-model", "input": "hello"},
cfg,
("guest", "fallback_test_embed"),
"test",
"default",
)
)
assert rt._client.calls[-1][1]["model"] == "qwen/qwen3-embedding-8b"
def test_images_unknown_model_falls_back_to_default(local_db, monkeypatch):
monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeImageClient_openai_gateway)
svc = GatewayService()
cfg = svc.effective_config()
cfg["gateway_force_model"] = False
cfg["gateway_image_enabled"] = True
cfg["gateway_image_model"] = "black-forest-labs/flux.2-pro"
rt = svc.runtime()
run_async(
rt.handle_images(
{"model": "nonexistent-image-model", "prompt": "test badge"},
cfg,
("guest", "fallback_test_img"),
"test",
"default",
)
)
assert rt._client.calls[-1][1]["model"] == "black-forest-labs/flux.2-pro"
class FakeClientWithUsage_openai_gateway(FakeClient_openai_gateway):
async def send(self, request, stream=False):
self.calls.append((request.url, request.json_body))
body = request.json_body or {}
return FakeResp_openai_gateway(
payload={
"id": "x",
"model": body.get("model"),
"choices": [{"message": {"content": "hi there"}}],
"usage": {"prompt_tokens": 7, "completion_tokens": 3, "total_tokens": 10},
}
)
def test_ollama_stream_does_not_send_stream_options(local_db, monkeypatch):
monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeClient_openai_gateway)
svc = GatewayService()
cfg = svc.effective_config()
cfg["gateway_upstream_url"] = "http://127.0.0.1:11434/api/chat"
rt = svc.runtime()
run_async(
rt.handle_chat(
{
"messages": [{"role": "user", "content": "hi"}],
"stream": True,
"stream_options": {"include_usage": True},
},
cfg,
("guest", "ollama_stream_opts"),
"test",
"default",
)
)
upstream_payload = rt._client.calls[-1][1]
assert upstream_payload["stream"] is True
assert "stream_options" not in upstream_payload
assert upstream_payload["think"] is False
def test_ollama_explicit_dialect_overrides_url(local_db, monkeypatch):
monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeClient_openai_gateway)
svc = GatewayService()
cfg = svc.effective_config()
cfg["gateway_upstream_url"] = "https://ai.example.com/v1/chat/completions"
cfg["gateway_thinking_dialect"] = "ollama"
rt = svc.runtime()
run_async(
rt.handle_chat(
{"messages": [{"role": "user", "content": "hi"}]},
cfg,
("guest", "ollama_dialect_override"),
"test",
"default",
)
)
body = rt._client.calls[-1][1]
assert body["think"] is False
assert "thinking" not in body
def test_client_model_allowed_when_allow_client_model_on(local_db, monkeypatch):
monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeClient_openai_gateway)
svc = GatewayService()
cfg = svc.effective_config()
cfg["gateway_force_model"] = True
cfg["gateway_allow_client_model"] = True
cfg["gateway_model"] = "deepseek-chat"
rt = svc.runtime()
run_async(
rt.handle_chat(
{"model": "llama3.2", "messages": [{"role": "user", "content": "hi"}]},
cfg,
("guest", "client_model"),
"test",
"default",
)
)
assert rt._client.calls[-1][1]["model"] == "llama3.2"
def test_molodetz_alias_still_remapped_with_allow_client_model(local_db, monkeypatch):
from devplacepy.services.openai_gateway import routing
monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeClient_openai_gateway)
original = routing.model_store.get("molodetz")
routing.model_store.remove("molodetz")
try:
svc = GatewayService()
cfg = svc.effective_config()
cfg["gateway_force_model"] = True
cfg["gateway_allow_client_model"] = True
cfg["gateway_model"] = "deepseek-chat"
rt = svc.runtime()
run_async(
rt.handle_chat(
{"model": "molodetz", "messages": [{"role": "user", "content": "hi"}]},
cfg,
("guest", "molodetz_alias"),
"test",
"default",
)
)
assert rt._client.calls[-1][1]["model"] == "deepseek-chat"
finally:
if original is not None:
routing.model_store.set(
routing.ModelRouteIn(
**{
field: getattr(original, field)
for field in routing.ModelRouteIn.model_fields
}
)
)
def test_stream_forwarded_to_upstream_with_forced_include_usage(local_db, monkeypatch):
monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeClient_openai_gateway)
svc = GatewayService()
cfg = svc.effective_config()
rt = svc.runtime()
run_async(
rt.handle_chat(
{
"messages": [{"role": "user", "content": "hi"}],
"stream": True,
"stream_options": {"include_usage": False},
},
cfg,
("guest", "test"),
"test",
"default",
)
)
upstream_payload = rt._client.calls[-1][1]
assert upstream_payload["stream"] is True
assert upstream_payload["stream_options"] == {"include_usage": True}
assert upstream_payload["thinking"] == {"type": "disabled"}
assert "think" not in upstream_payload
assert "reasoning" not in upstream_payload
def test_non_stream_has_no_stream_options_upstream(local_db, monkeypatch):
monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeClient_openai_gateway)
svc = GatewayService()
cfg = svc.effective_config()
rt = svc.runtime()
run_async(
rt.handle_chat(
{
"messages": [{"role": "user", "content": "hi"}],
"stream_options": {"include_usage": True},
},
cfg,
("guest", "test"),
"test",
"default",
)
)
upstream_payload = rt._client.calls[-1][1]
assert "stream_options" not in upstream_payload
assert upstream_payload["stream"] is False
def test_thinking_disabled_by_default(local_db, monkeypatch):
monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeClient_openai_gateway)
svc = GatewayService()
cfg = svc.effective_config()
rt = svc.runtime()
run_async(
rt.handle_chat(
{"messages": [{"role": "user", "content": "hi"}]},
cfg,
("guest", "test"),
"test",
"default",
)
)
body = rt._client.calls[-1][1]
assert body["thinking"] == {"type": "disabled"}
def test_client_think_true_enables_upstream_thinking(local_db, monkeypatch):
monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeClient_openai_gateway)
svc = GatewayService()
cfg = svc.effective_config()
rt = svc.runtime()
run_async(
rt.handle_chat(
{
"messages": [{"role": "user", "content": "hi"}],
"think": True,
},
cfg,
("guest", "test"),
"test",
"default",
)
)
body = rt._client.calls[-1][1]
assert body["thinking"] == {"type": "enabled"}
assert "think" not in body
def test_openrouter_upstream_gets_reasoning_none(local_db, monkeypatch):
monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeClient_openai_gateway)
svc = GatewayService()
cfg = svc.effective_config()
cfg["gateway_upstream_url"] = "https://openrouter.ai/api/v1/chat/completions"
rt = svc.runtime()
run_async(
rt.handle_chat(
{"messages": [{"role": "user", "content": "hi"}]},
cfg,
("guest", "test"),
"test",
"default",
)
)
body = rt._client.calls[-1][1]
assert body["reasoning"] == {"effort": "none"}
assert "thinking" not in body
def test_include_usage_emits_usage_chunk(local_db, monkeypatch):
monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeClientWithUsage_openai_gateway)
svc = GatewayService()
cfg = svc.effective_config()
rt = svc.runtime()
resp = run_async(
rt.handle_chat(
{
"messages": [{"role": "user", "content": "hi"}],
"stream": True,
"stream_options": {"include_usage": True},
},
cfg,
("guest", "test"),
"test",
"default",
)
)
async def drain():
out = []
async for chunk in resp.body_iterator:
out.append(chunk if isinstance(chunk, str) else chunk.decode())
return "".join(out)
body = run_async(drain())
assert '"usage"' in body
assert '"total_tokens": 10' in body
assert "[DONE]" in body
def test_no_usage_chunk_without_include_usage(local_db, monkeypatch):
monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeClientWithUsage_openai_gateway)
svc = GatewayService()
cfg = svc.effective_config()
rt = svc.runtime()
resp = run_async(
rt.handle_chat(
{"messages": [{"role": "user", "content": "hi"}], "stream": True},
cfg,
("guest", "test"),
"test",
"default",
)
)
async def drain():
out = []
async for chunk in resp.body_iterator:
out.append(chunk if isinstance(chunk, str) else chunk.decode())
return "".join(out)
body = run_async(drain())
assert '"usage"' not in body
assert "[DONE]" in body
def test_stream_headers_carry_no_cost_or_token_data(local_db, monkeypatch):
monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeClientWithUsage_openai_gateway)
svc = GatewayService()
cfg = svc.effective_config()
rt = svc.runtime()
resp = run_async(
rt.handle_chat(
{"messages": [{"role": "user", "content": "hi"}], "stream": True},
cfg,
("guest", "test"),
"test",
"my-app",
)
)
assert resp.headers["X-Gateway-Model"] == cfg["gateway_model"]
assert resp.headers["X-Gateway-Backend"] == "chat"
assert resp.headers["X-App-Reference"] == "my-app"
assert "x-gateway-cost-usd" not in {k.lower() for k in resp.headers}
assert "x-gateway-total-tokens" not in {k.lower() for k in resp.headers}
def test_stream_records_ttft_and_inter_token_in_ledger(local_db, monkeypatch):
monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeClientWithUsage_openai_gateway)
svc = GatewayService()
cfg = svc.effective_config()
rt = svc.runtime()
resp = run_async(
rt.handle_chat(
{
"messages": [{"role": "user", "content": "hi"}],
"stream": True,
"stream_options": {"include_usage": True},
},
cfg,
("guest", "ttft_probe"),
"test",
"default",
)
)
async def drain():
async for _ in resp.body_iterator:
pass
run_async(drain())
row = get_table("gateway_usage_ledger").find_one(owner_id="ttft_probe")
assert row is not None
assert row["backend"] == "chat"
assert row["stream_requested"] == 1
assert row["success"] == 1
assert row["total_tokens"] == 10
assert row["ttft_ms"] is not None and row["ttft_ms"] >= 0.0
assert row["inter_token_ms"] is not None and row["inter_token_ms"] >= 0.0
class FakeClientReasoning_openai_gateway(FakeClient_openai_gateway):
async def send(self, request, stream=False):
self.calls.append((request.url, request.json_body))
body = request.json_body or {}
return FakeResp_openai_gateway(
payload={
"id": "x",
"model": body.get("model"),
"choices": [{"message": {"content": "x", "reasoning": "aaaaa"}}],
"usage": {"prompt_tokens": 7, "completion_tokens": 3, "total_tokens": 10},
}
)
def test_ollama_reasoning_delta_is_counted_as_a_content_chunk(local_db, monkeypatch):
monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeClientReasoning_openai_gateway)
svc = GatewayService()
cfg = svc.effective_config()
rt = svc.runtime()
resp = run_async(
rt.handle_chat(
{
"messages": [{"role": "user", "content": "hi"}],
"stream": True,
"stream_options": {"include_usage": True},
},
cfg,
("guest", "ollama_reasoning_probe"),
"test",
"default",
)
)
async def drain():
async for _ in resp.body_iterator:
pass
run_async(drain())
row = get_table("gateway_usage_ledger").find_one(owner_id="ollama_reasoning_probe")
assert row is not None
assert row["inter_token_ms"] is not None and row["inter_token_ms"] >= 0.0
def test_stream_client_disconnect_records_failure_and_closes_upstream(local_db, monkeypatch):
monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeClient_openai_gateway)
svc = GatewayService()
cfg = svc.effective_config()
rt = svc.runtime()
resp = run_async(
rt.handle_chat(
{"messages": [{"role": "user", "content": "hi"}], "stream": True},
cfg,
("guest", "disconnect_probe"),
"test",
"default",
)
)
async def partial_then_close():
agen = resp.body_iterator
await agen.__anext__()
await agen.aclose()
run_async(partial_then_close())
row = get_table("gateway_usage_ledger").find_one(owner_id="disconnect_probe")
assert row is not None
assert row["success"] == 0
assert row["error_category"] == "client_disconnected"
def test_models_endpoint_publishes_molodetz(local_db):
from devplacepy.services.openai_gateway import routing
routing.seed_default_deepseek_routes()
try:
svc = GatewayService()
resp = svc._models_response()
payload = json.loads(resp.body.decode())
ids = {m["id"] for m in payload["data"]}
assert payload["object"] == "list"
assert {"molodetz", "molodetz-pro"} <= ids
for model in payload["data"]:
assert model["object"] == "model"
assert model["owned_by"] == "molodetz"
finally:
routing.model_store.remove("molodetz")
routing.model_store.remove("molodetz-pro")
# Upstream header collision policy: only X-Gateway-Model is ever trusted and
# forwarded from an upstream that happens to speak our own X-Gateway-* header
# convention (e.g. another DevPlace-style gateway). Every other figure
# (cost/tokens/latency/context/app-reference) must remain ours regardless of
# what an upstream claims. See usage.upstream_reported_model / CLAUDE.md.
class FakeClientUpstreamModelHeader_openai_gateway(FakeClient_openai_gateway):
async def send(self, request, stream=False):
self.calls.append((request.url, request.json_body))
body = request.json_body or {}
return FakeResp_openai_gateway(
payload={
"id": "x",
"model": body.get("model"),
"choices": [{"message": {"content": "hi there"}}],
},
extra_headers={"X-Gateway-Model": "upstream/served-model-x"},
)
class FakeClientHostileModelHeader_openai_gateway(FakeClient_openai_gateway):
async def send(self, request, stream=False):
self.calls.append((request.url, request.json_body))
body = request.json_body or {}
return FakeResp_openai_gateway(
payload={
"id": "x",
"model": body.get("model"),
"choices": [{"message": {"content": "hi there"}}],
},
extra_headers={"X-Gateway-Model": "evil\r\nX-Injected: true"},
)
class FakeEmbedClientUpstreamModelHeader_openai_gateway(FakeEmbedClient_openai_gateway):
async def send(self, request, stream=False):
self.calls.append((request.url, request.json_body))
body = request.json_body or {}
return FakeResp_openai_gateway(
payload={
"object": "list",
"model": body.get("model"),
"data": [{"object": "embedding", "index": 0, "embedding": [0.1, 0.2]}],
"usage": {"prompt_tokens": 5, "total_tokens": 5},
},
extra_headers={"X-Gateway-Model": "upstream/embed-served"},
)
class FakeImageClientUpstreamModelHeader_openai_gateway(FakeImageClient_openai_gateway):
async def send(self, request, stream=False):
self.calls.append((request.url, request.json_body))
body = request.json_body or {}
return FakeResp_openai_gateway(
payload={
"created": 1,
"model": body.get("model"),
"data": [{"b64_json": "aGVsbG8="}],
"usage": {"cost": 0.05},
},
extra_headers={"X-Gateway-Model": "upstream/image-served"},
)
class FakePassthroughClientUpstreamModelHeader_openai_gateway:
def __init__(self, *a, **k):
self.calls = []
def build_request(self, method, url, headers=None, json=None, content=None):
return FakeRequest(method, url, json)
async def send(self, request, stream=False):
self.calls.append((request.url, request.json_body))
return FakeResp_openai_gateway(
payload={"ok": True},
extra_headers={"X-Gateway-Model": "upstream/passthrough-served"},
)
async def aclose(self):
pass
def test_upstream_x_gateway_model_header_is_forwarded_for_chat(local_db, monkeypatch):
monkeypatch.setattr(
gwmod.httpx, "AsyncClient", FakeClientUpstreamModelHeader_openai_gateway
)
svc = GatewayService()
cfg = svc.effective_config()
cfg["gateway_force_model"] = True
cfg["gateway_model"] = "deepseek-chat"
rt = svc.runtime()
resp = run_async(
rt.handle_chat(
{"messages": [{"role": "user", "content": "hi"}]},
cfg,
("guest", "upstream_model_chat"),
"test",
"default",
)
)
assert resp.headers["X-Gateway-Model"] == "upstream/served-model-x"
row = get_table("gateway_usage_ledger").find_one(owner_id="upstream_model_chat")
assert row is not None
assert row["model"] == "deepseek-chat"
def test_upstream_x_gateway_model_header_is_forwarded_for_streaming_chat(
local_db, monkeypatch
):
monkeypatch.setattr(
gwmod.httpx, "AsyncClient", FakeClientUpstreamModelHeader_openai_gateway
)
svc = GatewayService()
cfg = svc.effective_config()
rt = svc.runtime()
resp = run_async(
rt.handle_chat(
{"messages": [{"role": "user", "content": "hi"}], "stream": True},
cfg,
("guest", "upstream_model_stream"),
"test",
"default",
)
)
assert resp.headers["X-Gateway-Model"] == "upstream/served-model-x"
assert resp.headers["X-Gateway-Backend"] == "chat"
async def drain():
async for _ in resp.body_iterator:
pass
run_async(drain())
def test_upstream_x_gateway_model_header_is_forwarded_for_embeddings(
local_db, monkeypatch
):
monkeypatch.setattr(
gwmod.httpx, "AsyncClient", FakeEmbedClientUpstreamModelHeader_openai_gateway
)
monkeypatch.setenv("OPENROUTER_API_KEY", "test-embed-key")
svc = GatewayService()
cfg = svc.effective_config()
cfg["gateway_embed_enabled"] = True
cfg["gateway_embed_model"] = "qwen/qwen3-embedding-8b"
rt = svc.runtime()
resp = run_async(
rt.handle_embeddings(
{"input": "hello"},
cfg,
("guest", "upstream_model_embed"),
"test",
"default",
)
)
assert resp.headers["X-Gateway-Model"] == "upstream/embed-served"
row = get_table("gateway_usage_ledger").find_one(owner_id="upstream_model_embed")
assert row is not None
assert row["model"] == "qwen/qwen3-embedding-8b"
def test_upstream_x_gateway_model_header_is_forwarded_for_images(local_db, monkeypatch):
monkeypatch.setattr(
gwmod.httpx, "AsyncClient", FakeImageClientUpstreamModelHeader_openai_gateway
)
svc = GatewayService()
cfg = svc.effective_config()
cfg["gateway_image_enabled"] = True
cfg["gateway_image_model"] = "black-forest-labs/flux.2-pro"
rt = svc.runtime()
resp = run_async(
rt.handle_images(
{"prompt": "badge"},
cfg,
("guest", "upstream_model_img"),
"test",
"default",
)
)
assert resp.headers["X-Gateway-Model"] == "upstream/image-served"
row = get_table("gateway_usage_ledger").find_one(owner_id="upstream_model_img")
assert row is not None
assert row["model"] == "black-forest-labs/flux.2-pro"
def test_upstream_x_gateway_model_header_is_forwarded_for_passthrough(
local_db, monkeypatch
):
monkeypatch.setattr(
gwmod.httpx, "AsyncClient", FakePassthroughClientUpstreamModelHeader_openai_gateway
)
svc = GatewayService()
cfg = svc.effective_config()
cfg["gateway_model"] = "deepseek-chat"
rt = svc.runtime()
resp = run_async(
rt.handle_passthrough(
"POST",
"responses",
"application/json",
b"{}",
cfg,
("guest", "upstream_model_passthrough"),
"test",
"default",
)
)
assert resp.headers["X-Gateway-Model"] == "upstream/passthrough-served"
row = get_table("gateway_usage_ledger").find_one(
owner_id="upstream_model_passthrough"
)
assert row is not None
assert row["model"] == "deepseek-chat"
def test_no_upstream_model_header_keeps_our_own_model(local_db, monkeypatch):
monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeClient_openai_gateway)
svc = GatewayService()
cfg = svc.effective_config()
cfg["gateway_force_model"] = True
cfg["gateway_model"] = "deepseek-chat"
rt = svc.runtime()
resp = run_async(
rt.handle_chat(
{"messages": [{"role": "user", "content": "hi"}]},
cfg,
("guest", "no_upstream_header"),
"test",
"default",
)
)
assert resp.headers["X-Gateway-Model"] == "deepseek-chat"
class FakeEmbeddedErrorClient_openai_gateway:
"""Mimics OpenRouter's documented behavior of answering 200 OK with the
failure embedded in the JSON body (openrouter.ai/docs/api-reference/errors)
instead of a non-2xx status."""
def __init__(self, *a, **k):
self.calls = []
def build_request(self, method, url, headers=None, json=None, content=None):
return FakeRequest(method, url, json)
async def send(self, request, stream=False):
self.calls.append((request.url, request.json_body))
body = request.json_body or {}
model = body.get("model")
if model == "primary-target":
return FakeResp_openai_gateway(
status=200,
payload={"error": {"message": "no provider available", "code": 502}},
)
return FakeResp_openai_gateway(
payload={
"id": "x",
"model": model,
"choices": [{"message": {"content": "fallback ok"}}],
}
)
async def aclose(self):
pass
def test_chat_falls_back_when_upstream_returns_200_with_an_embedded_error(
local_db, monkeypatch
):
from devplacepy.services.openai_gateway import routing
monkeypatch.setattr(
gwmod.httpx, "AsyncClient", FakeEmbeddedErrorClient_openai_gateway
)
routing.model_store.set(
routing.ModelRouteIn(source_model="eb-backup-route", target_model="backup-target")
)
routing.model_store.set(
routing.ModelRouteIn(
source_model="eb-primary-route",
target_model="primary-target",
fallback_model="eb-backup-route",
)
)
try:
svc = GatewayService()
cfg = svc.effective_config()
cfg["gateway_vision_enabled"] = False
cfg["gateway_max_retries"] = 0
rt = svc.runtime()
response = run_async(
rt.handle_chat(
{"model": "eb-primary-route", "messages": [{"role": "user", "content": "hi"}]},
cfg,
("guest", "embedded_error_chat"),
"test",
"default",
)
)
assert response.status_code == 200
assert rt._client.calls[0][1]["model"] == "primary-target"
assert rt._client.calls[-1][1]["model"] == "backup-target"
row = get_table("gateway_usage_ledger").find_one(owner_id="embedded_error_chat")
assert row is not None
assert row["requested_model"] == "eb-primary-route"
assert row["model"] == "backup-target"
assert row["success"] == 1
finally:
routing.model_store.remove("eb-primary-route")
routing.model_store.remove("eb-backup-route")
class FakeAlwaysEmbeddedErrorClient_openai_gateway:
def __init__(self, *a, **k):
self.calls = []
def build_request(self, method, url, headers=None, json=None, content=None):
return FakeRequest(method, url, json)
async def send(self, request, stream=False):
self.calls.append((request.url, request.json_body))
return FakeResp_openai_gateway(
status=200,
payload={"error": {"message": "no provider available", "code": 502}},
)
async def aclose(self):
pass
def test_chat_reports_502_when_upstream_returns_200_with_an_embedded_error_and_no_fallback(
local_db, monkeypatch
):
monkeypatch.setattr(
gwmod.httpx, "AsyncClient", FakeAlwaysEmbeddedErrorClient_openai_gateway
)
svc = GatewayService()
cfg = svc.effective_config()
cfg["gateway_vision_enabled"] = False
cfg["gateway_max_retries"] = 0
rt = svc.runtime()
response = run_async(
rt.handle_chat(
{"messages": [{"role": "user", "content": "hi"}]},
cfg,
("guest", "embedded_error_no_fallback"),
"test",
"default",
)
)
assert response.status_code == 502
row = get_table("gateway_usage_ledger").find_one(owner_id="embedded_error_no_fallback")
assert row is not None
assert row["success"] == 0
assert row["error_category"] == "upstream_error"
def test_ollama_dialect_sends_reasoning_effort_alongside_think(local_db, monkeypatch):
# Ollama's native /api/chat honors a boolean `think`, but its
# OpenAI-compatible /v1/chat/completions layer (what this gateway
# actually calls) does not - it maps reasoning_effort/reasoning
# instead (github.com/ollama/ollama issues #15288, #15293, #14820).
monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeClient_openai_gateway)
svc = GatewayService()
cfg = svc.effective_config()
cfg["gateway_upstream_url"] = "http://127.0.0.1:11434/v1/chat/completions"
rt = svc.runtime()
run_async(
rt.handle_chat(
{"messages": [{"role": "user", "content": "hi"}]},
cfg,
("guest", "ollama_reasoning_effort"),
"test",
"default",
)
)
body = rt._client.calls[-1][1]
assert body["think"] is False
assert body["reasoning_effort"] == "none"
def test_hostile_upstream_model_header_is_ignored_end_to_end(local_db, monkeypatch):
monkeypatch.setattr(
gwmod.httpx, "AsyncClient", FakeClientHostileModelHeader_openai_gateway
)
svc = GatewayService()
cfg = svc.effective_config()
cfg["gateway_force_model"] = True
cfg["gateway_model"] = "deepseek-chat"
rt = svc.runtime()
resp = run_async(
rt.handle_chat(
{"messages": [{"role": "user", "content": "hi"}]},
cfg,
("guest", "hostile_header"),
"test",
"default",
)
)
assert resp.headers["X-Gateway-Model"] == "deepseek-chat"
def test_bypass_preamble_skips_the_operator_preamble_when_allowed(local_db, monkeypatch):
monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeClient_openai_gateway)
svc = GatewayService()
cfg = svc.effective_config()
cfg["gateway_system_preamble"] = "OPERATOR-PREAMBLE-TEXT"
rt = svc.runtime()
run_async(
rt.handle_chat(
{
"messages": [
{"role": "system", "content": "Reply with strict JSON only."},
{"role": "user", "content": "hi"},
],
"bypass_preamble": True,
},
cfg,
("internal", "devii"),
"test",
"default",
bypass_allowed=True,
)
)
sent = rt._client.calls[-1][1]
system_message = sent["messages"][0]
assert system_message["role"] == "system"
assert "OPERATOR-PREAMBLE-TEXT" not in system_message["content"]
assert "Current date" in system_message["content"]
assert "Reply with strict JSON only." in system_message["content"]
assert "bypass_preamble" not in sent
def test_bypass_preamble_ignored_without_internal_credentials(local_db, monkeypatch):
monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeClient_openai_gateway)
svc = GatewayService()
cfg = svc.effective_config()
cfg["gateway_system_preamble"] = "OPERATOR-PREAMBLE-TEXT"
rt = svc.runtime()
run_async(
rt.handle_chat(
{"messages": [{"role": "user", "content": "hi"}], "bypass_preamble": True},
cfg,
("user", "someuser"),
"test",
"default",
bypass_allowed=False,
)
)
sent = rt._client.calls[-1][1]
system_message = sent["messages"][0]
assert "OPERATOR-PREAMBLE-TEXT" in system_message["content"]
assert "bypass_preamble" not in sent
def test_bypass_preamble_keeps_the_clients_own_system_content(local_db, monkeypatch):
monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeClient_openai_gateway)
svc = GatewayService()
cfg = svc.effective_config()
cfg["gateway_system_preamble"] = "OPERATOR-PREAMBLE-TEXT"
rt = svc.runtime()
run_async(
rt.handle_chat(
{
"messages": [
{"role": "system", "content": "Grade strictly. Reply JSON only."},
{"role": "user", "content": "hi"},
],
"bypass_preamble": True,
},
cfg,
("internal", "devii"),
"test",
"default",
bypass_allowed=True,
)
)
sent = rt._client.calls[-1][1]
system_message = sent["messages"][0]
assert "OPERATOR-PREAMBLE-TEXT" not in system_message["content"]
assert "Grade strictly. Reply JSON only." in system_message["content"]
def test_internal_bypass_allowed_requires_the_exact_internal_key(local_db):
set_setting("gateway_internal_key", "test-internal-secret")
try:
svc = GatewayService()
cfg = svc.effective_config()
assert (
svc.internal_bypass_allowed(
_make_request_openai_gateway(
headers={"X-Gateway-Internal-Key": "test-internal-secret"}
),
cfg,
)
is True
)
assert (
svc.internal_bypass_allowed(
_make_request_openai_gateway(
headers={"X-Gateway-Internal-Key": "wrong-secret"}
),
cfg,
)
is False
)
assert (
svc.internal_bypass_allowed(
_make_request_openai_gateway(
headers={"Authorization": "Bearer test-internal-secret"}
),
cfg,
)
is False
)
assert svc.internal_bypass_allowed(_make_request_openai_gateway(), cfg) is False
finally:
set_setting("gateway_internal_key", "")
class FakeResponsesResp_openai_gateway(FakeResp_openai_gateway):
def __init__(self, status=200, payload=None, events=None, **kw):
super().__init__(status=status, payload=payload, **kw)
self._events = events or []
async def aiter_lines(self):
for event in self._events:
yield f"data: {json.dumps(event)}"
yield "data: [DONE]"
def _resp_obj_openai_gateway(model):
return {
"id": "resp_1",
"object": "response",
"created_at": 7,
"status": "completed",
"model": model,
"output": [
{
"type": "message",
"role": "assistant",
"content": [{"type": "output_text", "text": "hi there"}],
}
],
"usage": {"input_tokens": 5, "output_tokens": 7, "total_tokens": 12},
}
def _resp_events_openai_gateway(model):
return [
{"type": "response.created", "response": {"id": "resp_1"}},
{"type": "response.output_text.delta", "delta": "hi "},
{"type": "response.output_text.delta", "delta": "there"},
{"type": "response.completed", "response": _resp_obj_openai_gateway(model)},
]
def _resp_tool_obj_openai_gateway(model):
obj = _resp_obj_openai_gateway(model)
obj["output"].append(
{
"type": "function_call",
"call_id": "call_7",
"name": "ext_bash",
"arguments": "{}",
}
)
return obj
def _resp_tool_events_openai_gateway(model):
return [
{"type": "response.created", "response": {"id": "resp_1"}},
{
"type": "response.output_item.added",
"output_index": 1,
"item": {
"type": "function_call",
"call_id": "call_7",
"name": "ext_bash",
"arguments": "",
},
},
{
"type": "response.function_call_arguments.delta",
"item_id": "call_7",
"output_index": 1,
"delta": "{}",
},
{
"type": "response.completed",
"response": _resp_tool_obj_openai_gateway(model),
},
]
class FakeResponsesClient_openai_gateway:
behavior = "ok"
def __init__(self, *a, **k):
self.calls = []
self.last_headers = {}
def build_request(self, method, url, headers=None, json=None, content=None):
return FakeRequest(method, url, json, headers)
async def send(self, request, stream=False):
self.calls.append((request.url, request.json_body))
self.last_headers = dict(request.headers or {})
body = request.json_body or {}
if self.behavior == "error-envelope":
return FakeResp_openai_gateway(
status=429,
payload={
"error": {
"message": "slow down",
"type": "rate_limit",
"code": "rate_limit_exceeded",
}
},
)
if self.behavior == "failed-object":
return FakeResponsesResp_openai_gateway(
payload={
"object": "response",
"status": "failed",
"error": {"message": "boom"},
}
)
if self.behavior == "attestation-denied":
return FakeResp_openai_gateway(
status=403,
payload={
"error": {
"message": "This API key is billed to a Muse Code "
"subscription. Muse Code subscriptions cover requests "
"from the Muse Code client only.",
"type": "permission_error",
"code": "client_attestation_denied",
}
},
)
model = body.get("model", "")
if self.behavior == "tool-call":
return FakeResponsesResp_openai_gateway(
payload=_resp_tool_obj_openai_gateway(model),
events=_resp_tool_events_openai_gateway(model),
)
return FakeResponsesResp_openai_gateway(
payload=_resp_obj_openai_gateway(model),
events=_resp_events_openai_gateway(model),
)
async def post(self, url, headers=None, json=None, timeout=None):
return await self.send(FakeRequest("POST", url, json), stream=False)
async def aclose(self):
pass
def _setup_responses_route_openai_gateway():
from devplacepy.services.openai_gateway import routing
routing.provider_store.set(
routing.ProviderIn(
name="gwmeta",
base_url="https://api.meta.ai/v1/responses",
api_key="sk-test",
protocol="responses",
)
)
routing.model_store.set(
routing.ModelRouteIn(
source_model="meta-chat",
provider="gwmeta",
target_model="muse-spark-1.3-contributor",
kind="chat",
)
)
def _teardown_responses_route_openai_gateway():
from devplacepy.services.openai_gateway import routing
routing.model_store.remove("meta-chat")
routing.provider_store.remove("gwmeta")
def _chat_body_openai_gateway(**over):
body = {
"model": "meta-chat",
"messages": [
{"role": "system", "content": "Be terse."},
{"role": "user", "content": "hi"},
],
}
body.update(over)
return body
def test_responses_protocol_transpiles_request_and_response(local_db, monkeypatch):
monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeResponsesClient_openai_gateway)
FakeResponsesClient_openai_gateway.behavior = "ok"
_setup_responses_route_openai_gateway()
try:
svc = GatewayService()
cfg = svc.effective_config()
cfg["gateway_vision_enabled"] = False
rt = svc.runtime()
resp = run_async(
rt.handle_chat(
_chat_body_openai_gateway(), cfg, ("guest", "test"), "test", "default"
)
)
url, sent = rt._client.calls[-1]
assert str(url) == "https://api.meta.ai/v1/responses"
assert sent["model"] == "muse-spark-1.3-contributor"
assert "messages" not in sent
assert "Be terse." in sent["instructions"]
assert sent["input"] == [
{"type": "message", "role": "user", "content": "hi"}
]
assert sent["reasoning"] == {"effort": "minimal"}
assert resp.status_code == 200
payload = json.loads(bytes(resp.body).decode())
assert payload["object"] == "chat.completion"
assert payload["choices"][0]["message"]["content"] == "hi there"
assert payload["choices"][0]["finish_reason"] == "stop"
assert payload["usage"]["prompt_tokens"] == 5
assert resp.headers["X-Gateway-Model"] == "muse-spark-1.3-contributor"
finally:
_teardown_responses_route_openai_gateway()
def test_muse_profile_headers_reach_the_wire(local_db, monkeypatch):
from devplacepy.services.openai_gateway import routing
monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeResponsesClient_openai_gateway)
FakeResponsesClient_openai_gateway.behavior = "ok"
routing.provider_store.set(
routing.ProviderIn(
name="wiremeta",
base_url="https://api.meta.ai/v1/responses",
api_key="sk-test",
protocol="responses",
client_profile="muse",
)
)
routing.model_store.set(
routing.ModelRouteIn(
source_model="wire-muse",
provider="wiremeta",
target_model="muse-spark",
kind="chat",
)
)
try:
svc = GatewayService()
cfg = svc.effective_config()
cfg["gateway_vision_enabled"] = False
rt = svc.runtime()
logged = []
resp = run_async(
rt.handle_chat(
{"model": "wire-muse", "messages": [{"role": "user", "content": "hi"}]},
cfg,
("guest", "test"),
"test",
"default",
log=logged.append,
)
)
assert resp.status_code == 200
sent_headers = rt._client.last_headers
assert sent_headers["X-Client-Id"] == "tbh:exec"
assert sent_headers["User-Agent"].startswith("muse-build/")
session = sent_headers["X-Tbh-Session-Id"]
assert session
assert sent_headers["X-Meta-Ai-Gateway-Session-Id"] == session
assert sent_headers["X-Tbh-Thread-Id"] == session
assert sent_headers["X-Tbh-Root-Session-Id"] == session
assert any("(client profile 'muse')" in line for line in logged)
finally:
routing.model_store.remove("wire-muse")
routing.provider_store.remove("wiremeta")
def _bash_tool_body_openai_gateway(**over):
body = _chat_body_openai_gateway(**over)
body["tools"] = [
{
"type": "function",
"function": {
"name": "bash",
"description": "run shell",
"parameters": {"type": "object", "properties": {}},
},
}
]
return body
def test_meta_tool_names_aliased_on_wire_and_restored(local_db, monkeypatch):
monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeResponsesClient_openai_gateway)
FakeResponsesClient_openai_gateway.behavior = "tool-call"
_setup_responses_route_openai_gateway()
try:
svc = GatewayService()
cfg = svc.effective_config()
cfg["gateway_vision_enabled"] = False
rt = svc.runtime()
logged = []
resp = run_async(
rt.handle_chat(
_bash_tool_body_openai_gateway(),
cfg,
("guest", "test"),
"test",
"default",
log=logged.append,
)
)
_, sent = rt._client.calls[-1]
assert sent["tools"][0]["name"] == "ext_bash"
assert any("bash->ext_bash" in line for line in logged)
assert resp.status_code == 200
payload = json.loads(bytes(resp.body).decode())
calls = payload["choices"][0]["message"]["tool_calls"]
assert calls[0]["function"]["name"] == "bash"
assert payload["choices"][0]["finish_reason"] == "tool_calls"
finally:
FakeResponsesClient_openai_gateway.behavior = "ok"
_teardown_responses_route_openai_gateway()
def test_meta_tool_names_restored_on_stream_chunks(local_db, monkeypatch):
monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeResponsesClient_openai_gateway)
FakeResponsesClient_openai_gateway.behavior = "tool-call"
_setup_responses_route_openai_gateway()
try:
svc = GatewayService()
cfg = svc.effective_config()
cfg["gateway_vision_enabled"] = False
rt = svc.runtime()
resp = run_async(
rt.handle_chat(
_bash_tool_body_openai_gateway(stream=True),
cfg,
("guest", "test"),
"test",
"default",
)
)
async def drain():
out = []
async for chunk in resp.body_iterator:
out.append(chunk if isinstance(chunk, str) else chunk.decode())
return "".join(out)
body = run_async(drain())
assert '"name": "bash"' in body
assert "ext_bash" not in body
assert "[DONE]" in body
finally:
FakeResponsesClient_openai_gateway.behavior = "ok"
_teardown_responses_route_openai_gateway()
def _setup_chat_meta_route_openai_gateway():
from devplacepy.services.openai_gateway import routing
routing.provider_store.set(
routing.ProviderIn(
name="gwmetacc",
base_url="https://api.meta.ai/v1/chat/completions",
api_key="sk-test",
)
)
routing.model_store.set(
routing.ModelRouteIn(
source_model="meta-chat-cc",
provider="gwmetacc",
target_model="muse-spark",
kind="chat",
)
)
def _teardown_chat_meta_route_openai_gateway():
from devplacepy.services.openai_gateway import routing
routing.model_store.remove("meta-chat-cc")
routing.provider_store.remove("gwmetacc")
def _tai_tool_body_openai_gateway(**over):
# Mirrors a real tai/detz turn: core tools are always sent, so "shell"
# (a Pi-verified gate trigger missing from the denylist until
# 2026-11-10) rides every request alongside the lazy "read_file".
body = _chat_body_openai_gateway(**over)
body["model"] = "meta-chat-cc"
body["tools"] = [
{
"type": "function",
"function": {
"name": "shell",
"description": "run a shell command",
"parameters": {"type": "object", "properties": {}},
},
},
{
"type": "function",
"function": {
"name": "read_file",
"description": "read a file",
"parameters": {"type": "object", "properties": {}},
},
},
{
"type": "function",
"function": {
"name": "web_fetch",
"description": "fetch a URL",
"parameters": {"type": "object", "properties": {}},
},
},
]
return body
class FakeChatMetaStreamResp_openai_gateway(FakeResp_openai_gateway):
def __init__(self, payload):
super().__init__(status=200, payload=payload)
self._payload = payload
async def aiter_lines(self):
calls = self._payload["choices"][0]["message"]["tool_calls"]
yield (
'data: {"id":"chatcmpl-x","object":"chat.completion.chunk",'
'"model":"m","choices":[{"index":0,"delta":{"role":"assistant"}}]}'
)
for pos, call in enumerate(calls):
name = call["function"]["name"]
yield (
'data: {"id":"chatcmpl-x","object":"chat.completion.chunk",'
'"model":"m","choices":[{"index":0,"delta":{"tool_calls":'
'[{"index":%d,"id":"call_%d","function":{"name":"%s",'
'"arguments":"{}"}}]}}]}' % (pos, pos, name)
)
yield (
'data: {"id":"chatcmpl-x","object":"chat.completion.chunk",'
'"model":"m","choices":[{"index":0,"delta":{},'
'"finish_reason":"tool_calls"}]}'
)
yield "data: [DONE]"
class FakeChatMetaClient_openai_gateway:
def __init__(self, *a, **k):
self.calls = []
def build_request(self, method, url, headers=None, json=None, content=None):
return FakeRequest(method, url, json, headers)
async def send(self, request, stream=False):
self.calls.append((request.url, request.json_body))
body = request.json_body or {}
wire_names = [
(tool.get("function") or {}).get("name")
for tool in body.get("tools") or []
]
payload = {
"id": "chatcmpl-x",
"object": "chat.completion",
"model": body.get("model"),
"choices": [
{
"index": 0,
"message": {
"role": "assistant",
"content": "",
"tool_calls": [
{
"id": "call_7",
"type": "function",
"function": {"name": name, "arguments": "{}"},
}
for name in wire_names
],
},
"finish_reason": "tool_calls",
}
],
"usage": {"prompt_tokens": 5, "completion_tokens": 2, "total_tokens": 7},
}
if stream:
return FakeChatMetaStreamResp_openai_gateway(payload)
return FakeResp_openai_gateway(
payload=payload, content=json.dumps(payload).encode()
)
async def post(self, url, headers=None, json=None, timeout=None):
return await self.send(FakeRequest("POST", url, json), stream=False)
async def aclose(self):
pass
def test_meta_tool_names_aliased_on_chat_completions_wire(local_db, monkeypatch):
monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeChatMetaClient_openai_gateway)
_setup_chat_meta_route_openai_gateway()
try:
svc = GatewayService()
cfg = svc.effective_config()
cfg["gateway_vision_enabled"] = False
rt = svc.runtime()
logged = []
resp = run_async(
rt.handle_chat(
_tai_tool_body_openai_gateway(),
cfg,
("guest", "test"),
"test",
"default",
log=logged.append,
)
)
_, sent = rt._client.calls[-1]
wire = [tool["function"]["name"] for tool in sent["tools"]]
assert wire == ["ext_shell", "ext_read_file", "ext_web_fetch"]
assert any("shell->ext_shell" in line for line in logged)
assert any("read_file->ext_read_file" in line for line in logged)
assert any("web_fetch->ext_web_fetch" in line for line in logged)
assert resp.status_code == 200
payload = json.loads(bytes(resp.body).decode())
calls = payload["choices"][0]["message"]["tool_calls"]
assert [call["function"]["name"] for call in calls] == [
"shell",
"read_file",
"web_fetch",
]
assert payload["choices"][0]["finish_reason"] == "tool_calls"
finally:
_teardown_chat_meta_route_openai_gateway()
def test_meta_tool_names_restored_on_chat_completions_stream(
local_db, monkeypatch
):
monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeChatMetaClient_openai_gateway)
_setup_chat_meta_route_openai_gateway()
try:
svc = GatewayService()
cfg = svc.effective_config()
cfg["gateway_vision_enabled"] = False
rt = svc.runtime()
resp = run_async(
rt.handle_chat(
_tai_tool_body_openai_gateway(stream=True),
cfg,
("guest", "test"),
"test",
"default",
)
)
async def drain():
out = []
async for chunk in resp.body_iterator:
out.append(chunk if isinstance(chunk, str) else chunk.decode())
return "".join(out)
body = run_async(drain())
_, sent = rt._client.calls[-1]
wire = [tool["function"]["name"] for tool in sent["tools"]]
assert wire == ["ext_shell", "ext_read_file", "ext_web_fetch"]
assert '"name": "shell"' in body
assert '"name": "read_file"' in body
assert '"name": "web_fetch"' in body
assert "ext_shell" not in body
assert "ext_read_file" not in body
assert "ext_web_fetch" not in body
assert "[DONE]" in body
finally:
_teardown_chat_meta_route_openai_gateway()
def test_responses_protocol_stream_relays_chat_chunks(local_db, monkeypatch):
monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeResponsesClient_openai_gateway)
FakeResponsesClient_openai_gateway.behavior = "ok"
_setup_responses_route_openai_gateway()
try:
svc = GatewayService()
cfg = svc.effective_config()
cfg["gateway_vision_enabled"] = False
rt = svc.runtime()
resp = run_async(
rt.handle_chat(
_chat_body_openai_gateway(stream=True),
cfg,
("guest", "test"),
"test",
"default",
)
)
async def drain():
out = []
async for chunk in resp.body_iterator:
out.append(chunk if isinstance(chunk, str) else chunk.decode())
return "".join(out)
body = run_async(drain())
assert "chat.completion.chunk" in body
assert "hi " in body and "there" in body
assert '"usage"' not in body
assert "[DONE]" in body
finally:
_teardown_responses_route_openai_gateway()
def test_responses_protocol_stream_relays_usage_when_asked(local_db, monkeypatch):
monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeResponsesClient_openai_gateway)
FakeResponsesClient_openai_gateway.behavior = "ok"
_setup_responses_route_openai_gateway()
try:
svc = GatewayService()
cfg = svc.effective_config()
cfg["gateway_vision_enabled"] = False
rt = svc.runtime()
resp = run_async(
rt.handle_chat(
_chat_body_openai_gateway(
stream=True, stream_options={"include_usage": True}
),
cfg,
("guest", "test"),
"test",
"default",
)
)
_, sent = rt._client.calls[-1]
assert "stream_options" not in sent
async def drain():
out = []
async for chunk in resp.body_iterator:
out.append(chunk if isinstance(chunk, str) else chunk.decode())
return "".join(out)
body = run_async(drain())
assert '"usage"' in body
assert '"prompt_tokens": 5' in body
usage_chunks = [
json.loads(line[len("data:"):])
for line in body.splitlines()
if line.startswith("data: {") and '"usage"' in line
]
assert len(usage_chunks) == 1
assert usage_chunks[0]["choices"] == []
finally:
_teardown_responses_route_openai_gateway()
def test_responses_protocol_relays_error_envelope(local_db, monkeypatch):
monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeResponsesClient_openai_gateway)
FakeResponsesClient_openai_gateway.behavior = "error-envelope"
_setup_responses_route_openai_gateway()
try:
svc = GatewayService()
cfg = svc.effective_config()
cfg["gateway_vision_enabled"] = False
rt = svc.runtime()
resp = run_async(
rt.handle_chat(
_chat_body_openai_gateway(), cfg, ("guest", "test"), "test", "default"
)
)
assert resp.status_code == 429
payload = json.loads(bytes(resp.body).decode())
assert payload["error"]["message"] == "slow down"
assert payload["error"]["code"] == "rate_limit_exceeded"
finally:
FakeResponsesClient_openai_gateway.behavior = "ok"
_teardown_responses_route_openai_gateway()
def test_responses_protocol_attestation_denied_logs_guidance(
local_db, monkeypatch
):
monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeResponsesClient_openai_gateway)
FakeResponsesClient_openai_gateway.behavior = "attestation-denied"
_setup_responses_route_openai_gateway()
try:
svc = GatewayService()
cfg = svc.effective_config()
cfg["gateway_vision_enabled"] = False
rt = svc.runtime()
logged = []
resp = run_async(
rt.handle_chat(
_chat_body_openai_gateway(),
cfg,
("guest", "test"),
"test",
"default",
log=logged.append,
)
)
assert resp.status_code == 403
payload = json.loads(bytes(resp.body).decode())
assert payload["error"]["code"] == "client_attestation_denied"
assert any("pay-as-you-go" in line for line in logged)
finally:
FakeResponsesClient_openai_gateway.behavior = "ok"
_teardown_responses_route_openai_gateway()
def test_upstream_auth_guidance_covers_401_and_403():
denied = FakeResp_openai_gateway(
status=403,
payload={
"error": {
"message": "subscription only",
"type": "permission_error",
"code": "client_attestation_denied",
}
},
)
guidance = gwmod._upstream_auth_guidance(denied, denied.json())
assert "pay-as-you-go" in guidance
assert "aliasing" in guidance
assert "MUSE_RESERVED_TOOL_NAMES" in guidance
other_denied = FakeResp_openai_gateway(status=403, payload={"error": "nope"})
assert (
gwmod._upstream_auth_guidance(other_denied, other_denied.json()) is None
)
unauthed = FakeResp_openai_gateway(
status=401, payload={"error": {"message": "User not found.", "code": 401}}
)
guidance = gwmod._upstream_auth_guidance(unauthed, unauthed.json())
assert "api_key" in guidance
assert "User not found." in guidance
server_error = FakeResp_openai_gateway(status=500, payload={"error": "x"})
assert (
gwmod._upstream_auth_guidance(server_error, server_error.json()) is None
)
def test_should_fallback_model_unavailable_but_not_bad_request():
unavailable = FakeResp_openai_gateway(
status=400,
payload={"error": {"message": "Model is unavailable."}},
)
assert (
gwmod._should_fallback(unavailable, None, {"circuit_open": False}) is True
)
malformed = FakeResp_openai_gateway(
status=400, payload={"error": "bad schema"}
)
assert (
gwmod._should_fallback(malformed, None, {"circuit_open": False}) is False
)
def test_responses_protocol_failed_object_is_502(local_db, monkeypatch):
monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeResponsesClient_openai_gateway)
FakeResponsesClient_openai_gateway.behavior = "failed-object"
_setup_responses_route_openai_gateway()
try:
svc = GatewayService()
cfg = svc.effective_config()
cfg["gateway_vision_enabled"] = False
rt = svc.runtime()
resp = run_async(
rt.handle_chat(
_chat_body_openai_gateway(), cfg, ("guest", "test"), "test", "default"
)
)
assert resp.status_code == 502
finally:
FakeResponsesClient_openai_gateway.behavior = "ok"
_teardown_responses_route_openai_gateway()
def test_responses_protocol_client_reasoning_effort_passes_through(
local_db, monkeypatch
):
monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeResponsesClient_openai_gateway)
FakeResponsesClient_openai_gateway.behavior = "ok"
_setup_responses_route_openai_gateway()
try:
svc = GatewayService()
cfg = svc.effective_config()
cfg["gateway_vision_enabled"] = False
rt = svc.runtime()
run_async(
rt.handle_chat(
_chat_body_openai_gateway(reasoning_effort="high"),
cfg,
("guest", "test"),
"test",
"default",
)
)
_, sent = rt._client.calls[-1]
assert sent["reasoning"] == {"effort": "high"}
finally:
_teardown_responses_route_openai_gateway()
def test_responses_protocol_default_provider(local_db, monkeypatch):
monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeResponsesClient_openai_gateway)
FakeResponsesClient_openai_gateway.behavior = "ok"
svc = GatewayService()
cfg = svc.effective_config()
cfg["gateway_vision_enabled"] = False
cfg["gateway_upstream_url"] = "https://api.meta.ai/v1/responses"
cfg["gateway_upstream_protocol"] = "responses"
cfg["gateway_model"] = "muse-spark-1.3-contributor"
rt = svc.runtime()
resp = run_async(
rt.handle_chat(
{"messages": [{"role": "user", "content": "hi"}]},
cfg,
("guest", "test"),
"test",
"default",
)
)
url, sent = rt._client.calls[-1]
assert str(url) == "https://api.meta.ai/v1/responses"
assert sent["input"] == [{"type": "message", "role": "user", "content": "hi"}]
assert resp.status_code == 200
payload = json.loads(bytes(resp.body).decode())
assert payload["object"] == "chat.completion"
EMULATED_CALL_TEXT_openai_gateway = (
"Sure.\n<tool_call>\n"
'{"name": "bash", "arguments": {"command": "ls"}}\n'
"</tool_call>"
)
BASH_TOOL_openai_gateway = {
"type": "function",
"function": {
"name": "bash",
"description": "Run a shell command",
"parameters": {
"type": "object",
"properties": {"command": {"type": "string"}},
"required": ["command"],
},
},
}
def _emulated_obj_openai_gateway(model, text):
obj = _resp_obj_openai_gateway(model)
obj["output"][0]["content"][0]["text"] = text
return obj
class FakeEmulatedClient_openai_gateway(FakeResponsesClient_openai_gateway):
text = EMULATED_CALL_TEXT_openai_gateway
async def send(self, request, stream=False):
self.calls.append((request.url, request.json_body))
model = (request.json_body or {}).get("model", "")
obj = _emulated_obj_openai_gateway(model, self.text)
events = [{"type": "response.created", "response": {"id": "resp_1"}}]
for i in range(0, len(self.text), 4):
events.append(
{"type": "response.output_text.delta", "delta": self.text[i : i + 4]}
)
events.append({"type": "response.completed", "response": obj})
return FakeResponsesResp_openai_gateway(payload=obj, events=events)
def _setup_emulated_route_openai_gateway():
from devplacepy.services.openai_gateway import routing
routing.provider_store.set(
routing.ProviderIn(
name="gwmeta",
base_url="https://api.meta.ai/v1/responses",
api_key="sk-test",
protocol="responses",
)
)
routing.model_store.set(
routing.ModelRouteIn(
source_model="meta-chat",
provider="gwmeta",
target_model="muse-spark-1.3-contributor",
kind="chat",
tools_mode="emulated",
)
)
def _emulated_chat_openai_gateway(monkeypatch, body, **tools):
monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeEmulatedClient_openai_gateway)
_setup_emulated_route_openai_gateway()
svc = GatewayService()
cfg = svc.effective_config()
cfg["gateway_vision_enabled"] = False
rt = svc.runtime()
resp = run_async(rt.handle_chat(body, cfg, ("guest", "test"), "test", "default"))
return rt, resp
def test_emulated_tools_meta_responses_nonstream(local_db, monkeypatch):
body = _chat_body_openai_gateway(tools=[BASH_TOOL_openai_gateway], tool_choice="auto")
try:
rt, resp = _emulated_chat_openai_gateway(monkeypatch, body)
url, sent = rt._client.calls[-1]
assert str(url) == "https://api.meta.ai/v1/responses"
assert "tools" not in sent and "tool_choice" not in sent
assert "<tools>" in sent["instructions"] and '"bash"' in sent["instructions"]
assert resp.status_code == 200
message = json.loads(bytes(resp.body).decode())["choices"][0]
assert message["finish_reason"] == "tool_calls"
call = message["message"]["tool_calls"][0]
assert call["function"]["name"] == "bash"
assert json.loads(call["function"]["arguments"]) == {"command": "ls"}
assert message["message"]["content"] == "Sure."
finally:
_teardown_responses_route_openai_gateway()
def test_emulated_tools_meta_responses_stream(local_db, monkeypatch):
body = _chat_body_openai_gateway(
stream=True, tools=[BASH_TOOL_openai_gateway]
)
try:
rt, resp = _emulated_chat_openai_gateway(monkeypatch, body)
async def drain():
out = []
async for chunk in resp.body_iterator:
out.append(chunk if isinstance(chunk, str) else chunk.decode())
return "".join(out)
raw = run_async(drain())
chunks = [
json.loads(line[5:])
for line in raw.splitlines()
if line.startswith("data:") and "[DONE]" not in line
]
text = "".join(
c["choices"][0]["delta"].get("content", "")
for c in chunks
if c.get("choices")
)
calls = [
tc
for c in chunks
for choice in c.get("choices") or []
for tc in choice["delta"].get("tool_calls") or []
]
assert "<tool_call>" not in text and text.strip() == "Sure."
assert len(calls) == 1 and calls[0]["function"]["name"] == "bash"
finishes = [
c["choices"][0]["finish_reason"]
for c in chunks
if c.get("choices") and c["choices"][0].get("finish_reason")
]
assert finishes[-1] == "tool_calls"
assert raw.rstrip().endswith("[DONE]")
finally:
_teardown_responses_route_openai_gateway()
def test_emulated_tools_convert_history_for_the_meta_transpiler(local_db, monkeypatch):
history = [
{"role": "user", "content": "go"},
{
"role": "assistant",
"content": None,
"tool_calls": [
{
"id": "call_1",
"type": "function",
"function": {"name": "bash", "arguments": '{"command": "ls"}'},
}
],
},
{"role": "tool", "tool_call_id": "call_1", "content": "a.txt"},
]
body = _chat_body_openai_gateway(messages=history, tools=[BASH_TOOL_openai_gateway])
try:
rt, resp = _emulated_chat_openai_gateway(monkeypatch, body)
_, sent = rt._client.calls[-1]
assert resp.status_code == 200
assert all(item.get("type") == "message" for item in sent["input"])
joined = json.dumps(sent["input"])
assert "<tool_call>" in joined and "<tool_response>" in joined and "a.txt" in joined
assert "function_call" not in joined
finally:
_teardown_responses_route_openai_gateway()
def test_emulated_tools_serve_large_catalogs_with_no_cap(local_db, monkeypatch):
tools = [
{
"type": "function",
"function": {
"name": f"tool_{index}",
"description": f"tool number {index}",
"parameters": {"type": "object", "properties": {}, "required": []},
},
}
for index in range(200)
]
body = _chat_body_openai_gateway(tools=tools, tool_choice="auto")
try:
rt, resp = _emulated_chat_openai_gateway(monkeypatch, body)
assert resp.status_code == 200
assert len(rt._client.calls) == 1
_, sent = rt._client.calls[-1]
assert "tools" not in sent and "tool_choice" not in sent
assert '"name": "tool_0"' in sent["instructions"]
assert '"name": "tool_199"' in sent["instructions"]
assert len(rt._catalog_cache._entries) == 1
finally:
_teardown_responses_route_openai_gateway()
def test_native_route_keeps_forwarding_tools(local_db, monkeypatch):
monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeResponsesClient_openai_gateway)
FakeResponsesClient_openai_gateway.behavior = "ok"
_setup_responses_route_openai_gateway()
try:
svc = GatewayService()
cfg = svc.effective_config()
cfg["gateway_vision_enabled"] = False
rt = svc.runtime()
run_async(
rt.handle_chat(
_chat_body_openai_gateway(tools=[BASH_TOOL_openai_gateway]),
cfg,
("guest", "test"),
"test",
"default",
)
)
_, sent = rt._client.calls[-1]
assert "tools" in sent
assert "<tools>" not in sent["instructions"]
finally:
_teardown_responses_route_openai_gateway()
VAIDS_REJECTION_openai_gateway = {
"details": {"_errors": ["Unrecognized key(s) in object: 'thinking'"]},
"error": "Invalid request parameters",
"issues": [
{
"code": "unrecognized_keys",
"keys": ["thinking"],
"path": [],
"message": "Unrecognized key(s) in object: 'thinking'",
}
],
}
def _rejection_openai_gateway(key):
return {
"details": {"_errors": [f"Unrecognized key(s) in object: '{key}'"]},
"error": "Invalid request parameters",
"issues": [{"code": "unrecognized_keys", "keys": [key], "path": []}],
}
class FakeStrictClient_openai_gateway:
rejected = ("thinking",)
def __init__(self, *a, **k):
self.calls = []
def build_request(self, method, url, headers=None, json=None, content=None):
return FakeRequest(method, url, json)
async def send(self, request, stream=False):
body = request.json_body or {}
self.calls.append((request.url, dict(body)))
for key in self.rejected:
if key in body:
return FakeResp_openai_gateway(
status=400, payload=_rejection_openai_gateway(key)
)
return FakeResp_openai_gateway(
payload={
"id": "x",
"model": body.get("model"),
"choices": [{"message": {"content": "hi there"}}],
}
)
async def aclose(self):
pass
class FakeStubbornClient_openai_gateway(FakeStrictClient_openai_gateway):
async def send(self, request, stream=False):
self.calls.append((request.url, dict(request.json_body or {})))
return FakeResp_openai_gateway(
status=400, payload=_rejection_openai_gateway("thinking")
)
class FakeReasoningStrictClient_openai_gateway(FakeStrictClient_openai_gateway):
rejected = ("reasoning",)
def _shape_cfg_openai_gateway():
svc = GatewayService()
cfg = svc.effective_config()
cfg["gateway_vision_enabled"] = False
cfg["gateway_max_retries"] = 0
return svc, cfg
def _chat_openai_gateway(rt, cfg, owner, stream=False):
body = {"messages": [{"role": "user", "content": "hi"}]}
if stream:
body["stream"] = True
return run_async(rt.handle_chat(body, cfg, ("guest", owner), "test", "default"))
def test_chat_auto_resolves_a_rejected_thinking_key_and_remembers_it(local_db, monkeypatch):
from devplacepy.services.openai_gateway.learned import learned_store
monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeStrictClient_openai_gateway)
svc, cfg = _shape_cfg_openai_gateway()
model = cfg["gateway_model"]
learned_store.forget("default", model)
try:
rt = svc.runtime()
response = _chat_openai_gateway(rt, cfg, "shape_resolve_first")
assert response.status_code == 200
assert len(rt._client.calls) == 2
assert "thinking" in rt._client.calls[0][1]
assert "thinking" not in rt._client.calls[1][1]
learned = learned_store.get("default", model)
assert learned is not None and learned.thinking_dialect == "none"
row = get_table("gateway_usage_ledger").find_one(owner_id="shape_resolve_first")
assert row["resolved_method"] == "none"
assert row["success"] == 1
second = _chat_openai_gateway(rt, cfg, "shape_resolve_second")
assert second.status_code == 200
assert len(rt._client.calls) == 3
assert "thinking" not in rt._client.calls[2][1]
finally:
learned_store.forget("default", model)
def test_chat_auto_resolve_works_for_streams(local_db, monkeypatch):
from devplacepy.services.openai_gateway.learned import learned_store
monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeStrictClient_openai_gateway)
svc, cfg = _shape_cfg_openai_gateway()
model = cfg["gateway_model"]
learned_store.forget("default", model)
try:
rt = svc.runtime()
response = _chat_openai_gateway(rt, cfg, "shape_resolve_stream", stream=True)
async def drain():
out = []
async for chunk in response.body_iterator:
out.append(chunk if isinstance(chunk, str) else chunk.decode())
return "".join(out)
assert "[DONE]" in run_async(drain())
assert len(rt._client.calls) == 2
assert learned_store.get("default", model) is not None
finally:
learned_store.forget("default", model)
def test_chat_genuine_bad_request_is_not_auto_resolved(local_db, monkeypatch):
from devplacepy.services.openai_gateway.learned import learned_store
monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeBadRequestClient_openai_gateway)
svc, cfg = _shape_cfg_openai_gateway()
model = cfg["gateway_model"]
learned_store.forget("default", model)
rt = svc.runtime()
response = _chat_openai_gateway(rt, cfg, "shape_genuine_bad")
assert response.status_code == 400
assert len(rt._client.calls) == 1
assert learned_store.get("default", model) is None
def test_chat_exhausted_resolution_returns_the_original_error_and_stores_nothing(
local_db, monkeypatch
):
from devplacepy.services.openai_gateway.learned import learned_store
from devplacepy.services.openai_gateway.shape import MAX_RESOLVE_ATTEMPTS
monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeStubbornClient_openai_gateway)
svc, cfg = _shape_cfg_openai_gateway()
model = cfg["gateway_model"]
learned_store.forget("default", model)
rt = svc.runtime()
response = _chat_openai_gateway(rt, cfg, "shape_exhausted")
assert response.status_code == 400
assert "thinking" in response.body.decode()
assert len(rt._client.calls) <= 1 + MAX_RESOLVE_ATTEMPTS
assert learned_store.get("default", model) is None
def test_chat_rejected_learned_method_is_forgotten_and_ladder_restarts(local_db, monkeypatch):
from devplacepy.services.openai_gateway.learned import learned_store
from devplacepy.services.openai_gateway.shape import ShapeMethod
monkeypatch.setattr(
gwmod.httpx, "AsyncClient", FakeReasoningStrictClient_openai_gateway
)
svc, cfg = _shape_cfg_openai_gateway()
model = cfg["gateway_model"]
learned_store.remember("default", model, ShapeMethod("openrouter"))
try:
rt = svc.runtime()
response = _chat_openai_gateway(rt, cfg, "shape_heal")
assert response.status_code == 200
assert "reasoning" in rt._client.calls[0][1]
assert "thinking" in rt._client.calls[1][1]
assert learned_store.get("default", model) is None
finally:
learned_store.forget("default", model)
def test_fallback_attempt_is_shaped_with_its_own_learned_method(local_db, monkeypatch):
from devplacepy.services.openai_gateway import routing
from devplacepy.services.openai_gateway.learned import learned_store
from devplacepy.services.openai_gateway.shape import ShapeMethod
class FakeFallbackShapeClient(FakeStrictClient_openai_gateway):
rejected = ()
async def send(self, request, stream=False):
body = request.json_body or {}
self.calls.append((request.url, dict(body)))
if body.get("model") == "shape-primary-target":
return FakeResp_openai_gateway(status=503, payload={"error": {"message": "down"}})
return FakeResp_openai_gateway(
payload={
"id": "x",
"model": body.get("model"),
"choices": [{"message": {"content": "ok"}}],
}
)
monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeFallbackShapeClient)
routing.model_store.set(
routing.ModelRouteIn(source_model="shape-backup", target_model="shape-backup-target")
)
routing.model_store.set(
routing.ModelRouteIn(
source_model="shape-primary",
target_model="shape-primary-target",
fallback_model="shape-backup",
)
)
learned_store.remember("default", "shape-backup-target", ShapeMethod("none"))
try:
svc, cfg = _shape_cfg_openai_gateway()
rt = svc.runtime()
response = run_async(
rt.handle_chat(
{"model": "shape-primary", "messages": [{"role": "user", "content": "hi"}]},
cfg,
("guest", "shape_fallback"),
"test",
"default",
)
)
assert response.status_code == 200
assert "thinking" in rt._client.calls[0][1]
assert rt._client.calls[-1][1]["model"] == "shape-backup-target"
assert "thinking" not in rt._client.calls[-1][1]
finally:
learned_store.forget("default", "shape-backup-target")
routing.model_store.remove("shape-primary")
routing.model_store.remove("shape-backup")
def _target_send_openai_gateway(rt, cfg, client, url, model):
import asyncio
async def go():
return await rt._send(
client,
asyncio.Semaphore(4),
"POST",
url,
{},
{**cfg, "gateway_max_retries": 0},
lambda message: None,
json_body={"model": model},
)
return run_async(go())
def _target_runtime_openai_gateway():
svc = GatewayService()
return svc.runtime(), svc.effective_config()
def test_trip_opens_below_threshold_and_ignores_threshold():
from devplacepy.services.openai_gateway.reliability import CircuitBreaker
breaker = CircuitBreaker(5, 30)
breaker.trip()
assert breaker.is_open is True
assert breaker.allow() is False
def test_a_failed_target_is_skipped_without_an_upstream_call(local_db):
rt, cfg = _target_runtime_openai_gateway()
client = FakeAlwaysFailClient_openai_gateway()
url = "https://target-a.example/v1/chat/completions"
first = _target_send_openai_gateway(rt, cfg, client, url, "m1")
assert first[0].status_code == 500
second = _target_send_openai_gateway(rt, cfg, client, url, "m1")
assert second[2]["circuit_open"] is True
assert len(client.calls) == 1
def test_one_bad_request_does_not_block_another_target(local_db):
rt, cfg = _target_runtime_openai_gateway()
client = FakeAlwaysFailClient_openai_gateway()
url = "https://target-b.example/v1/chat/completions"
_target_send_openai_gateway(rt, cfg, client, url, "model-b1")
other = _target_send_openai_gateway(rt, cfg, client, url, "model-b2")
assert other[2]["circuit_open"] is False
assert len(client.calls) == 2
def test_target_breaker_kill_switch_keeps_global_behaviour(local_db):
rt, cfg = _target_runtime_openai_gateway()
cfg = {**cfg, "gateway_target_circuit_enabled": False}
client = FakeAlwaysFailClient_openai_gateway()
url = "https://target-d.example/v1/chat/completions"
_target_send_openai_gateway(rt, cfg, client, url, "m1")
again = _target_send_openai_gateway(rt, cfg, client, url, "m1")
assert again[2]["circuit_open"] is False
assert len(client.calls) == 2
def test_shape_fixable_400_does_not_trip_the_target(local_db):
rt, cfg = _target_runtime_openai_gateway()
resp = FakeResp_openai_gateway(
status=400,
payload={"issues": [{"code": "unrecognized_keys", "keys": ["thinking"]}]},
)
assert gwmod._trip_worthy(resp) is False
plain = FakeResp_openai_gateway(status=400, payload={"error": "bad schema"})
assert gwmod._trip_worthy(plain) is True
def test_embedded_200_error_trips_the_target(local_db):
rt, cfg = _target_runtime_openai_gateway()
resp = FakeResp_openai_gateway(status=200, payload={"error": {"message": "provider exploded"}})
url = "https://target-e.example/v1/chat/completions"
assert rt._embedded_error(resp, False, url, "m1", cfg) is not None
assert rt._target_breaker("m1", cfg).is_open is True
assert rt.open_targets()[0]["model"] == "m1"
def test_transition_notifies_each_admin_once_with_detail_and_no_key(local_db, monkeypatch):
sent = []
monkeypatch.setattr(gwmod.background, "submit", lambda fn, *a, **k: sent.append((fn, a)))
rt, cfg = _target_runtime_openai_gateway()
url = "https://target-f.example/v1/chat/completions"
rt._trip_target("m1", url, cfg, "upstream_error", 500, "Bearer sk-secret123 failed", 1)
rt._trip_target("m1", url, cfg, "upstream_error", 500, "again", 1)
assert len(sent) == 1
detail = sent[0][1][1]
assert detail["category"] == "upstream_error"
assert detail["status"] == 500
assert "secret123" not in detail["upstream"]
assert detail["provider"] == "target-f.example"
def test_breaker_is_per_model_across_upstreams(local_db):
rt, cfg = _target_runtime_openai_gateway()
client = FakeAlwaysFailClient_openai_gateway()
_target_send_openai_gateway(rt, cfg, client, "https://target-g1.example/v1/chat/completions", "shared-model")
second = _target_send_openai_gateway(rt, cfg, client, "https://target-g2.example/v1/chat/completions", "shared-model")
assert second[2]["circuit_open"] is True
assert len(client.calls) == 1
def test_a_request_without_a_model_never_uses_the_target_breaker(local_db):
rt, cfg = _target_runtime_openai_gateway()
client = FakeAlwaysFailClient_openai_gateway()
url = "https://target-h.example/v1/other"
_target_send_openai_gateway(rt, cfg, client, url, "")
again = _target_send_openai_gateway(rt, cfg, client, url, "")
assert again[2]["circuit_open"] is False
assert rt._target_breakers == {}
class FakeNotifyClient_openai_gateway:
def __init__(self, *a, **k):
self.calls = []
def build_request(self, method, url, headers=None, json=None, content=None):
return FakeRequest(method, url, json)
async def send(self, request, stream=False):
self.calls.append((request.url, request.json_body))
body = request.json_body or {}
if body.get("model") == "nb-primary-target":
return FakeResp_openai_gateway(
status=500, payload={"error": "boom sk-secret-xyz"}
)
return FakeResp_openai_gateway(
payload={
"id": "x",
"model": body.get("model"),
"choices": [{"message": {"content": "notify ok"}}],
}
)
async def aclose(self):
pass
class FakeNotifyChainClient_openai_gateway:
def __init__(self, *a, **k):
self.calls = []
def build_request(self, method, url, headers=None, json=None, content=None):
return FakeRequest(method, url, json)
async def send(self, request, stream=False):
self.calls.append((request.url, request.json_body))
body = request.json_body or {}
if body.get("model") in ("n3-a-target", "n3-b-target"):
return FakeResp_openai_gateway(status=500, payload={"error": "boom"})
return FakeResp_openai_gateway(
payload={
"id": "x",
"model": body.get("model"),
"choices": [{"message": {"content": "chain ok"}}],
}
)
async def aclose(self):
pass
def _notify_chat_openai_gateway(monkeypatch, client_cls, routes, source, owner_id):
from devplacepy.services.openai_gateway import routing
monkeypatch.setattr(gwmod.httpx, "AsyncClient", client_cls)
for payload in routes:
routing.model_store.set(routing.ModelRouteIn(**payload))
sent = []
def _capture_notify_openai_gateway(fn, *a, **k):
if fn is gwmod._notify_gateway_status:
sent.append((fn, a))
monkeypatch.setattr(gwmod.background, "submit", _capture_notify_openai_gateway)
try:
svc = GatewayService()
cfg = svc.effective_config()
cfg["gateway_vision_enabled"] = False
cfg["gateway_max_retries"] = 0
rt = svc.runtime()
response = run_async(
rt.handle_chat(
{"model": source, "messages": [{"role": "user", "content": "hi"}]},
cfg,
("guest", owner_id),
"test",
"default",
)
)
return response, sent
finally:
for payload in routes:
routing.model_store.remove(payload["source_model"])
def _rendered_messages_openai_gateway(monkeypatch, details):
messages = []
class FakeUsersTable_openai_gateway:
def find(self, **kwargs):
if kwargs.get("role") == "Admin":
return [{"uid": "admin-1"}]
return []
monkeypatch.setattr(
"devplacepy.database.get_table",
lambda name: FakeUsersTable_openai_gateway(),
)
monkeypatch.setattr(
"devplacepy.utils.create_notification",
lambda uid, kind, message, *a, **k: messages.append(message),
)
for detail in details:
gwmod._notify_gateway_status(True, detail)
return messages
def test_trip_notification_carries_local_config_redirect_and_outcome(local_db, monkeypatch):
routes = [
{"source_model": "nb-backup", "target_model": "nb-backup-target"},
{
"source_model": "nb-primary",
"target_model": "nb-primary-target",
"fallback_model": "nb-backup",
},
]
response, sent = _notify_chat_openai_gateway(
monkeypatch, FakeNotifyClient_openai_gateway, routes, "nb-primary", "notify_one"
)
assert response.status_code == 200
assert len(sent) == 1
assert sent[0][0] is gwmod._notify_gateway_status
detail = sent[0][1][1]
assert detail["route"] == "nb-primary"
assert detail["model"] == "nb-primary-target"
assert detail["requested"] == "nb-primary"
assert "nb-primary" in detail["chain"]
assert "nb-backup" in detail["chain"]
assert detail["redirect"] == "nb-backup"
assert detail["outcome_ok"] is True
assert detail["served_route"] == "nb-backup"
assert "nb-backup" in detail["outcome"]
messages = _rendered_messages_openai_gateway(monkeypatch, [detail])
assert len(messages) == 1
assert "tripped its circuit breaker" in messages[0]
assert "Local config: 'nb-primary'" in messages[0]
assert "Full route:" in messages[0]
assert "nb-primary" in messages[0].split("Full route:")[1]
assert "nb-backup" in messages[0].split("Full route:")[1]
assert "Redirect: will redirect to fallback 'nb-backup'" in messages[0]
assert "Outcome: fallback 'nb-backup' answered HTTP 200 ok" in messages[0]
assert "Error class: upstream_error" in messages[0]
assert "HTTP status: 500" in messages[0]
assert "sk-secret-xyz" not in messages[0]
assert "[redacted]" in messages[0]
def test_trip_notifications_cover_every_hop_of_a_recursive_chain(local_db, monkeypatch):
routes = [
{"source_model": "n3-c", "target_model": "n3-c-target"},
{
"source_model": "n3-b",
"target_model": "n3-b-target",
"fallback_model": "n3-c",
},
{
"source_model": "n3-a",
"target_model": "n3-a-target",
"fallback_model": "n3-b",
},
]
response, sent = _notify_chat_openai_gateway(
monkeypatch, FakeNotifyChainClient_openai_gateway, routes, "n3-a", "notify_chain"
)
assert response.status_code == 200
assert len(sent) == 2
first = sent[0][1][1]
second = sent[1][1][1]
assert first["route"] == "n3-a"
assert first["redirect"] == "n3-b"
assert second["route"] == "n3-b"
assert second["redirect"] == "n3-c"
for detail in (first, second):
assert detail["requested"] == "n3-a"
assert "n3-a" in detail["chain"]
assert "n3-b" in detail["chain"]
assert "n3-c" in detail["chain"]
assert detail["outcome_ok"] is True
assert detail["outcome"] == "fallback 'n3-c' answered HTTP 200 ok"
assert detail["served_route"] == "n3-c"
messages = _rendered_messages_openai_gateway(monkeypatch, [first, second])
assert "Redirect: will redirect to fallback 'n3-b'" in messages[0]
assert "Redirect: will redirect to fallback 'n3-c'" in messages[1]
assert "Outcome: fallback 'n3-c' answered HTTP 200 ok" in messages[0]
assert "Outcome: fallback 'n3-c' answered HTTP 200 ok" in messages[1]
def test_trip_notification_reports_failed_outcome_when_every_fallback_fails(
local_db, monkeypatch
):
routes = [
{"source_model": "n4-b", "target_model": "n4-b-target"},
{
"source_model": "n4-a",
"target_model": "n4-a-target",
"fallback_model": "n4-b",
},
]
response, sent = _notify_chat_openai_gateway(
monkeypatch, FakeAlwaysFailClient_openai_gateway, routes, "n4-a", "notify_fail"
)
assert response.status_code == 500
assert len(sent) == 2
first = sent[0][1][1]
second = sent[1][1][1]
assert first["route"] == "n4-a"
assert first["redirect"] == "n4-b"
assert second["route"] == "n4-b"
assert second["redirect"] == ""
for detail in (first, second):
assert detail["outcome_ok"] is False
assert "all 2 route(s) failed" in detail["outcome"]
assert "HTTP 500" in detail["outcome"]
messages = _rendered_messages_openai_gateway(monkeypatch, [first, second])
assert "Redirect: will redirect to fallback 'n4-b'" in messages[0]
assert "Redirect: none, end of the fallback line" in messages[1]
assert "Outcome: all 2 route(s) failed" in messages[1]
def test_trip_notification_redirects_unrouted_default_to_first_fallback(
local_db, monkeypatch
):
rt, cfg = _target_runtime_openai_gateway()
sent = []
monkeypatch.setattr(
gwmod.background, "submit", lambda fn, *a, **k: sent.append((fn, a))
)
detail = gwmod._trip_detail(
"default-target",
"https://default.example/v1/chat/completions",
30,
"upstream_error",
500,
"boom",
0,
route="",
)
rt._notify_trips(
[{"kind": "target", "detail": detail}],
"chat",
"molodetz",
"weird-model",
[{"route": ""}, {"route": "m-pro"}],
True,
"fallback 'm-pro' answered HTTP 200 ok",
"m-pro",
)
assert len(sent) == 1
enriched = sent[0][1][1]
assert enriched["redirect"] == "m-pro"
messages = _rendered_messages_openai_gateway(monkeypatch, [enriched])
assert "Local config: default (no routed model)" in messages[0]
assert "Redirect: will redirect to fallback 'm-pro'" in messages[0]
assert "Outcome: fallback 'm-pro' answered HTTP 200 ok" in messages[0]
def test_vision_trip_notifies_with_local_config_and_no_fallback(local_db, monkeypatch):
import httpx
from devplacepy.services.openai_gateway.reliability import CircuitBreaker
from devplacepy.services.openai_gateway.vision import VisionAugmenter, VisionCache
class FailVisionClient_openai_gateway:
async def post(self, url, headers=None, json=None):
raise httpx.ConnectError("vision down")
breaker = CircuitBreaker(1, 60)
sink = []
augmenter = VisionAugmenter(
"https://vision.example/v1/chat/completions",
"vis-model",
"k",
VisionCache(0),
breaker=breaker,
max_retries=0,
trip_sink=sink,
)
image = {"type": "image_url", "image_url": {"url": "http://x/y.png"}}
first = run_async(augmenter._describe_one(FailVisionClient_openai_gateway(), image))
assert first.startswith("[vision call failed")
assert breaker.is_open is True
assert len(sink) == 1
assert sink[0]["kind"] == "vision"
assert sink[0]["detail"]["route"] == "vision:vis-model"
second = run_async(augmenter._describe_one(FailVisionClient_openai_gateway(), image))
assert second.startswith("[vision unavailable")
assert len(sink) == 1
rt, cfg = _target_runtime_openai_gateway()
sent = []
monkeypatch.setattr(
gwmod.background, "submit", lambda fn, *a, **k: sent.append((fn, a))
)
rt._notify_trips(
sink,
"chat",
"molodetz",
"molodetz",
[{"route": "molodetz"}],
True,
"fallback 'molodetz' answered HTTP 200 ok",
"molodetz",
)
assert len(sent) == 1
messages = _rendered_messages_openai_gateway(monkeypatch, [sent[0][1][1]])
assert "Local config: 'vision:vis-model'" in messages[0]
assert "Redirect: none, vision has no fallback model" in messages[0]
assert "Outcome: fallback 'molodetz' answered HTTP 200 ok" in messages[0]
def test_trip_notification_outcome_scrubs_secrets(local_db, monkeypatch):
detail = gwmod._trip_detail(
"s1-target",
"https://s1.example/v1/chat/completions",
30,
"connection",
"no response",
"boom",
0,
route="s1",
)
detail["requested"] = "s1"
detail["chain"] = "s1 (s1-target @ default)"
detail["redirect"] = ""
secret = "sk" + "-abc123"
detail["outcome"] = "connection failed (Bearer " + secret + ")"
messages = _rendered_messages_openai_gateway(monkeypatch, [detail])
assert secret not in messages[0]
assert "[redacted]" in messages[0]
def test_passthrough_global_trip_carries_local_config(local_db, monkeypatch):
monkeypatch.setattr(
gwmod.httpx, "AsyncClient", FakeAlwaysFailClient_openai_gateway
)
svc = GatewayService()
cfg = svc.effective_config()
cfg["gateway_circuit_threshold"] = 1
cfg["gateway_max_retries"] = 0
rt = svc.runtime()
sent = []
monkeypatch.setattr(
gwmod.background,
"submit",
lambda fn, *a, **k: sent.append((fn, a))
if fn is gwmod._notify_gateway_status
else None,
)
resp = run_async(
rt.handle_passthrough(
"POST",
"responses",
"application/json",
b"{}",
cfg,
("guest", "ptrip_local"),
"test",
"default",
)
)
assert resp.status_code == 500
assert len(sent) == 1
detail = sent[0][1][1]
assert detail.get("global") is True
assert detail["route"] == cfg["gateway_model"]
assert "no fallback line" in detail["chain"]
messages = _rendered_messages_openai_gateway(monkeypatch, [detail])
assert f"Local config: '{cfg['gateway_model']}'" in messages[0]
assert "passthrough POST responses (no fallback line)" in messages[0]
assert "Outcome: all 1 route(s) failed" in messages[0]