Files
devplacepy/tests/unit/services/openai_gateway/gateway.py
T
retoor 8a48e3ef85 Keep composers usable, group actions and notifications, and drop the tool-count cap
Comment and post layouts stay full width on small screens. Secondary actions live in the More menu. Notifications can be grouped and throttled, and Enter can submit when that profile setting is on. Emulated-tool models accept large catalogs, with reserved names aliased on the wire and the catalog cached per owner. A dropped service-worker navigation is retried once. The test server waits 120s to finish importing before it is stopped.
2026-11-12 15:29:27 +01:00

3444 lines
116 KiB
Python

# retoor <retoor@molodetz.nl>
import json
from starlette.requests import Request
from tests.conftest import BASE_URL, run_async
from devplacepy.database import get_table, set_setting
from devplacepy.utils import generate_uid
import devplacepy.services.openai_gateway.gateway as gwmod
from devplacepy.services.openai_gateway import GatewayService
from devplacepy.services.openai_gateway import auth_throttle
class FakeResp_openai_gateway:
def __init__(
self,
status=200,
payload=None,
ctype="application/json",
content=b"",
extra_headers=None,
):
self.status_code = status
self._payload = payload
self.text = json.dumps(payload) if payload is not None else ""
self.headers = {"content-type": ctype, **(extra_headers or {})}
self.content = content
def json(self):
if self._payload is None:
raise ValueError("no json")
return self._payload
async def aread(self):
return self.content or self.text.encode()
async def aclose(self):
pass
async def aiter_lines(self):
payload = self._payload or {}
chunk_id = payload.get("id", "x")
model = payload.get("model", "")
try:
content = payload["choices"][0]["message"].get("content") or ""
except (KeyError, IndexError, TypeError):
content = ""
try:
reasoning = payload["choices"][0]["message"].get("reasoning") or ""
except (KeyError, IndexError, TypeError):
reasoning = ""
def frame(delta=None, finish=None, usage=None):
body = {"id": chunk_id, "object": "chat.completion.chunk", "model": model}
if usage is not None:
body["choices"] = []
body["usage"] = usage
else:
body["choices"] = [{"index": 0, "delta": delta or {}, "finish_reason": finish}]
return f"data: {json.dumps(body)}"
yield frame({"role": "assistant"})
for i in range(0, len(reasoning), 5):
yield frame({"reasoning": reasoning[i : i + 5]})
for i in range(0, len(content), 5):
yield frame({"content": content[i : i + 5]})
yield frame({}, finish="stop")
if payload.get("usage"):
yield frame(usage=payload["usage"])
yield "data: [DONE]"
class FakeRequest:
def __init__(self, method, url, json_body, headers=None):
self.method = method
self.url = url
self.json_body = json_body
self.headers = headers or {}
self.extensions = {}
class FakeClient_openai_gateway:
def __init__(self, *a, **k):
self.calls = []
def build_request(self, method, url, headers=None, json=None, content=None):
return FakeRequest(method, url, json)
async def send(self, request, stream=False):
self.calls.append((request.url, request.json_body))
body = request.json_body or {}
return FakeResp_openai_gateway(
payload={
"id": "x",
"model": body.get("model"),
"choices": [{"message": {"content": "hi there"}}],
}
)
async def post(self, url, headers=None, json=None, timeout=None):
self.calls.append((url, json))
return FakeResp_openai_gateway(
payload={
"id": "x",
"model": json.get("model"),
"choices": [{"message": {"content": "hi there"}}],
}
)
async def request(self, method, url, headers=None, content=None):
return FakeResp_openai_gateway(payload={"ok": True})
async def aclose(self):
pass
def _make_request_openai_gateway(headers=None, cookies=None):
headers = headers or {}
raw = [(k.lower().encode(), v.encode()) for k, v in headers.items()]
if cookies:
raw.append(
(b"cookie", "; ".join(f"{k}={v}" for k, v in cookies.items()).encode())
)
return Request(
{
"type": "http",
"method": "POST",
"path": "/openai/v1/chat/completions",
"query_string": b"",
"headers": raw,
"state": {},
}
)
def _make_admin_openai_gateway(role="Admin"):
username = f"gw_{generate_uid()[:8]}"
api_key = generate_uid()
get_table("users").insert(
{
"uid": generate_uid(),
"username": username,
"terms_version": "1",
"email": f"{username}@t.dev",
"api_key": api_key,
"role": role,
"is_active": True,
}
)
return username, api_key
def _config(page, **fields):
page.request.post(f"{BASE_URL}/admin/services/openai/config", form=fields)
def test_model_is_forced(local_db, monkeypatch):
monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeClient_openai_gateway)
svc = GatewayService()
cfg = svc.effective_config()
cfg["gateway_force_model"] = True
cfg["gateway_model"] = "deepseek-chat"
rt = svc.runtime()
run_async(
rt.handle_chat(
{"model": "gpt-4", "messages": [{"role": "user", "content": "hi"}]},
cfg,
("guest", "test"),
"test",
"default",
)
)
assert rt._client.calls[-1][1]["model"] == "deepseek-chat"
def test_model_route_overrides_upstream(local_db, monkeypatch):
from devplacepy.services.openai_gateway import routing
monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeClient_openai_gateway)
routing.provider_store.set(
routing.ProviderIn(
name="gwroute",
base_url="https://routed.example/v1/chat/completions",
api_key="routed-key",
)
)
routing.model_store.set(
routing.ModelRouteIn(
source_model="custom-pro",
provider="gwroute",
target_model="vendor/pro",
kind="chat",
price_output_per_m=9.0,
)
)
try:
svc = GatewayService()
cfg = svc.effective_config()
cfg["gateway_vision_enabled"] = False
rt = svc.runtime()
run_async(
rt.handle_chat(
{"model": "custom-pro", "messages": [{"role": "user", "content": "hi"}]},
cfg,
("guest", "test"),
"test",
"default",
)
)
url, body = rt._client.calls[-1]
assert str(url) == "https://routed.example/v1/chat/completions"
assert body["model"] == "vendor/pro"
finally:
routing.model_store.remove("custom-pro")
routing.provider_store.remove("gwroute")
def test_vision_rewrites_image_to_text(local_db, monkeypatch):
monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeClient_openai_gateway)
svc = GatewayService()
cfg = svc.effective_config()
cfg["gateway_vision_enabled"] = True
cfg["gateway_vision_key"] = "" # no key -> placeholder text, still rewrites to str
rt = svc.runtime()
msgs = [
{
"role": "user",
"content": [
{"type": "text", "text": "what is this"},
{"type": "image_url", "image_url": {"url": "http://x/y.png"}},
],
}
]
run_async(rt.handle_chat({"messages": msgs}, cfg, ("guest", "test"), "test", "default"))
sent = rt._client.calls[-1][1]["messages"][0]["content"]
assert isinstance(sent, str) and "vision" in sent.lower()
def test_streaming_emits_sse(local_db, monkeypatch):
monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeClient_openai_gateway)
svc = GatewayService()
cfg = svc.effective_config()
rt = svc.runtime()
resp = run_async(
rt.handle_chat(
{"messages": [{"role": "user", "content": "hi"}], "stream": True},
cfg,
("guest", "test"),
"test",
"default",
)
)
async def drain():
out = []
async for chunk in resp.body_iterator:
out.append(chunk if isinstance(chunk, str) else chunk.decode())
return "".join(out)
body = run_async(drain())
assert "chat.completion.chunk" in body
assert "[DONE]" in body
def test_authorize_static_access_key(local_db):
set_setting("gateway_require_auth", "1")
set_setting("gateway_access_key", "topsecret")
try:
svc = GatewayService()
assert svc.authorize(_make_request_openai_gateway(headers={"X-API-KEY": "topsecret"})) is True
assert svc.authorize(_make_request_openai_gateway(headers={"X-API-KEY": "wrong"})) is False
finally:
set_setting("gateway_access_key", "")
set_setting("gateway_require_auth", "1")
auth_throttle.clear()
def test_authorize_admin_and_user_toggles(local_db):
set_setting("gateway_require_auth", "1")
set_setting("gateway_access_key", "")
set_setting("gateway_allow_admins", "1")
set_setting("gateway_allow_users", "0")
try:
_, admin_key = _make_admin_openai_gateway(role="Admin")
_, member_key = _make_admin_openai_gateway(role="Member")
svc = GatewayService()
assert (
svc.authorize(_make_request_openai_gateway(headers={"Authorization": f"Bearer {admin_key}"}))
is True
)
assert svc.authorize(_make_request_openai_gateway(headers={"X-API-KEY": member_key})) is False
set_setting("gateway_allow_users", "1")
assert svc.authorize(_make_request_openai_gateway(headers={"X-API-KEY": member_key})) is True
finally:
set_setting("gateway_allow_admins", "1")
set_setting("gateway_allow_users", "1")
set_setting("gateway_require_auth", "1")
set_setting("gateway_access_key", "")
auth_throttle.clear()
def test_authorize_require_auth_off_is_open(local_db):
set_setting("gateway_require_auth", "0")
try:
svc = GatewayService()
assert svc.authorize(_make_request_openai_gateway()) is True
finally:
set_setting("gateway_require_auth", "1")
def test_authorize_throttles_repeated_failed_auth_from_one_ip(local_db):
import pytest
from fastapi import HTTPException
set_setting("gateway_require_auth", "1")
set_setting("gateway_access_key", "")
set_setting("gateway_auth_throttle_max_failures", "3")
set_setting("gateway_auth_throttle_window_seconds", "60")
auth_throttle.clear()
try:
svc = GatewayService()
ip = "203.0.113.10"
for _ in range(3):
assert (
svc.authorize(_make_request_openai_gateway(headers={"X-Real-IP": ip}))
is False
)
with pytest.raises(HTTPException) as excinfo:
svc.authorize(_make_request_openai_gateway(headers={"X-Real-IP": ip}))
assert excinfo.value.status_code == 429
assert "Retry-After" in excinfo.value.headers
finally:
set_setting("gateway_auth_throttle_max_failures", "10")
set_setting("gateway_auth_throttle_window_seconds", "60")
set_setting("gateway_require_auth", "1")
auth_throttle.clear()
def test_authorize_throttle_does_not_affect_other_ips(local_db):
set_setting("gateway_require_auth", "1")
set_setting("gateway_access_key", "")
set_setting("gateway_auth_throttle_max_failures", "3")
set_setting("gateway_auth_throttle_window_seconds", "60")
auth_throttle.clear()
try:
svc = GatewayService()
throttled_ip = "203.0.113.20"
other_ip = "203.0.113.21"
for _ in range(3):
svc.authorize(_make_request_openai_gateway(headers={"X-Real-IP": throttled_ip}))
assert auth_throttle.is_throttled(throttled_ip, 3, 60) is True
assert (
svc.authorize(_make_request_openai_gateway(headers={"X-Real-IP": other_ip}))
is False
)
assert auth_throttle.is_throttled(other_ip, 3, 60) is False
finally:
set_setting("gateway_auth_throttle_max_failures", "10")
set_setting("gateway_auth_throttle_window_seconds", "60")
set_setting("gateway_require_auth", "1")
auth_throttle.clear()
def test_authorize_valid_credentials_succeed_from_throttled_ip(local_db):
set_setting("gateway_require_auth", "1")
set_setting("gateway_access_key", "")
set_setting("gateway_allow_users", "1")
set_setting("gateway_auth_throttle_max_failures", "3")
set_setting("gateway_auth_throttle_window_seconds", "60")
auth_throttle.clear()
try:
svc = GatewayService()
ip = "203.0.113.30"
_, member_key = _make_admin_openai_gateway(role="Member")
for _ in range(3):
svc.authorize(_make_request_openai_gateway(headers={"X-Real-IP": ip}))
assert auth_throttle.is_throttled(ip, 3, 60) is True
assert (
svc.authorize(
_make_request_openai_gateway(
headers={"X-Real-IP": ip, "X-API-KEY": member_key}
)
)
is True
)
set_setting("gateway_access_key", "topsecret")
assert (
svc.authorize(
_make_request_openai_gateway(
headers={"X-Real-IP": ip, "X-API-KEY": "topsecret"}
)
)
is True
)
finally:
set_setting("gateway_access_key", "")
set_setting("gateway_auth_throttle_max_failures", "10")
set_setting("gateway_auth_throttle_window_seconds", "60")
set_setting("gateway_require_auth", "1")
auth_throttle.clear()
def test_authorize_throttle_resets_after_window(local_db, monkeypatch):
set_setting("gateway_require_auth", "1")
set_setting("gateway_access_key", "")
set_setting("gateway_auth_throttle_max_failures", "3")
set_setting("gateway_auth_throttle_window_seconds", "60")
auth_throttle.clear()
try:
now = [2000.0]
monkeypatch.setattr(auth_throttle.time, "time", lambda: now[0])
svc = GatewayService()
ip = "203.0.113.40"
for _ in range(3):
svc.authorize(_make_request_openai_gateway(headers={"X-Real-IP": ip}))
assert auth_throttle.is_throttled(ip, 3, 60) is True
now[0] += 61
assert (
svc.authorize(_make_request_openai_gateway(headers={"X-Real-IP": ip}))
is False
)
finally:
set_setting("gateway_auth_throttle_max_failures", "10")
set_setting("gateway_auth_throttle_window_seconds", "60")
set_setting("gateway_require_auth", "1")
auth_throttle.clear()
class FakeEmbedClient_openai_gateway:
def __init__(self, *a, **k):
self.calls = []
def build_request(self, method, url, headers=None, json=None, content=None):
return FakeRequest(method, url, json)
async def send(self, request, stream=False):
self.calls.append((request.url, request.json_body))
body = request.json_body or {}
return FakeResp_openai_gateway(
payload={
"object": "list",
"model": body.get("model"),
"data": [{"object": "embedding", "index": 0, "embedding": [0.1, 0.2]}],
"usage": {"prompt_tokens": 5, "total_tokens": 5},
}
)
async def aclose(self):
pass
def test_embeddings_remaps_alias_to_configured_model(local_db, monkeypatch):
monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeEmbedClient_openai_gateway)
monkeypatch.setenv("OPENROUTER_API_KEY", "test-embed-key")
svc = GatewayService()
cfg = svc.effective_config()
cfg["gateway_force_model"] = False
cfg["gateway_embed_enabled"] = True
cfg["gateway_embed_model"] = "qwen/qwen3-embedding-8b"
rt = svc.runtime()
run_async(
rt.handle_embeddings(
{"model": "molodetz~embed", "input": "hello"},
cfg,
("guest", "test"),
"test",
"default",
)
)
assert rt._client.calls[-1][1]["model"] == "qwen/qwen3-embedding-8b"
def test_embeddings_success_records_ledger(local_db, monkeypatch):
monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeEmbedClient_openai_gateway)
monkeypatch.setenv("OPENROUTER_API_KEY", "test-embed-key")
svc = GatewayService()
cfg = svc.effective_config()
cfg["gateway_force_model"] = True
cfg["gateway_embed_enabled"] = True
cfg["gateway_embed_model"] = "qwen/qwen3-embedding-8b"
rt = svc.runtime()
before = rt.embed_calls
resp = run_async(
rt.handle_embeddings(
{"model": "molodetz~embed", "input": "hello"},
cfg,
("guest", "ledger_probe"),
"test",
"default",
)
)
assert resp.status_code == 200
payload = json.loads(bytes(resp.body).decode())
assert payload["data"][0]["embedding"] == [0.1, 0.2]
assert rt.embed_calls == before + 1
row = get_table("gateway_usage_ledger").find_one(owner_id="ledger_probe")
assert row is not None
assert row["backend"] == "embed"
assert row["endpoint"] == "embeddings"
assert row["requested_model"] == "molodetz~embed"
assert row["model"] == "qwen/qwen3-embedding-8b"
assert row["success"] == 1
def test_embeddings_disabled_returns_503(local_db, monkeypatch):
monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeEmbedClient_openai_gateway)
svc = GatewayService()
cfg = svc.effective_config()
cfg["gateway_embed_enabled"] = False
rt = svc.runtime()
resp = run_async(
rt.handle_embeddings(
{"input": "hello"}, cfg, ("guest", "test"), "test", "default"
)
)
assert resp.status_code == 503
def test_embeddings_missing_key_returns_clean_error_not_unauthenticated_call(
local_db, monkeypatch
):
monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeEmbedClient_openai_gateway)
monkeypatch.delenv("OPENROUTER_API_KEY", raising=False)
monkeypatch.delenv("DEEPSEEK_API_KEY", raising=False)
set_setting("gateway_api_key", "")
set_setting("gateway_vision_key", "")
set_setting("gateway_embed_key", "")
try:
svc = GatewayService()
cfg = svc.effective_config()
assert cfg["gateway_embed_key"] == ""
cfg["gateway_embed_enabled"] = True
rt = svc.runtime()
resp = run_async(
rt.handle_embeddings(
{"input": "hello"}, cfg, ("guest", "no_key_probe"), "test", "default"
)
)
assert resp.status_code == 503
payload = json.loads(bytes(resp.body).decode())
assert payload["error"]["type"] == "embeddings_not_configured"
assert rt._client is None
finally:
set_setting("gateway_api_key", "")
set_setting("gateway_vision_key", "")
set_setting("gateway_embed_key", "")
class FakeImageClient_openai_gateway:
def __init__(self, *a, **k):
self.calls = []
def build_request(self, method, url, headers=None, json=None, content=None):
return FakeRequest(method, url, json)
async def send(self, request, stream=False):
self.calls.append((request.url, request.json_body))
body = request.json_body or {}
return FakeResp_openai_gateway(
payload={
"created": 1,
"model": body.get("model"),
"data": [{"b64_json": "aGVsbG8="}],
"usage": {"cost": 0.05},
}
)
async def aclose(self):
pass
def test_images_remaps_alias_to_configured_model(local_db, monkeypatch):
monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeImageClient_openai_gateway)
svc = GatewayService()
cfg = svc.effective_config()
cfg["gateway_force_model"] = False
cfg["gateway_image_enabled"] = True
cfg["gateway_image_model"] = "black-forest-labs/flux.2-pro"
rt = svc.runtime()
run_async(
rt.handle_images(
{
"model": "molodetz-img-small",
"prompt": "award emblem",
"size": "512x512",
},
cfg,
("guest", "test"),
"test",
"default",
)
)
assert rt._client.calls[-1][1]["model"] == "black-forest-labs/flux.2-pro"
def test_image_route_overrides_upstream(local_db, monkeypatch):
from devplacepy.services.openai_gateway import routing
monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeImageClient_openai_gateway)
routing.provider_store.set(
routing.ProviderIn(
name="gwimg",
base_url="https://routed.example/v1/chat/completions",
api_key="img-key",
)
)
routing.model_store.set(
routing.ModelRouteIn(
source_model="custom-img",
provider="gwimg",
target_model="vendor/flux-pro",
kind="image",
price_input_per_m=0.06,
)
)
try:
svc = GatewayService()
cfg = svc.effective_config()
rt = svc.runtime()
run_async(
rt.handle_images(
{
"model": "custom-img",
"prompt": "trophy",
"response_format": "b64_json",
},
cfg,
("guest", "test"),
"test",
"default",
)
)
url, body = rt._client.calls[-1]
assert str(url) == "https://routed.example/v1/images"
assert body["model"] == "vendor/flux-pro"
finally:
routing.model_store.remove("custom-img")
routing.provider_store.remove("gwimg")
def test_images_success_records_ledger(local_db, monkeypatch):
monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeImageClient_openai_gateway)
svc = GatewayService()
cfg = svc.effective_config()
cfg["gateway_image_enabled"] = True
cfg["gateway_image_model"] = "black-forest-labs/flux.2-pro"
rt = svc.runtime()
before = rt.image_calls
resp = run_async(
rt.handle_images(
{"model": "molodetz-img-small", "prompt": "badge"},
cfg,
("guest", "img_ledger"),
"test",
"default",
)
)
assert resp.status_code == 200
payload = json.loads(bytes(resp.body).decode())
assert payload["data"][0]["b64_json"] == "aGVsbG8="
assert rt.image_calls == before + 1
row = get_table("gateway_usage_ledger").find_one(owner_id="img_ledger")
assert row is not None
assert row["backend"] == "image"
assert row["endpoint"] == "images/generations"
assert row["requested_model"] == "molodetz-img-small"
assert row["model"] == "black-forest-labs/flux.2-pro"
assert row["success"] == 1
assert float(row["cost_usd"]) == 0.05
def test_images_disabled_returns_503(local_db, monkeypatch):
monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeImageClient_openai_gateway)
svc = GatewayService()
cfg = svc.effective_config()
cfg["gateway_image_enabled"] = False
rt = svc.runtime()
resp = run_async(
rt.handle_images(
{"prompt": "badge"}, cfg, ("guest", "test"), "test", "default"
)
)
assert resp.status_code == 503
def test_compute_cost_embed_branch():
from devplacepy.services.openai_gateway.usage import (
Pricing,
compute_cost,
normalize_usage,
)
pricing = Pricing(
chat_cache_hit_per_m=0.0,
chat_cache_miss_per_m=0.0,
chat_output_per_m=0.0,
vision_input_per_m=0.0,
vision_output_per_m=0.0,
embed_input_per_m=0.01,
)
usage = {"prompt_tokens": 1_000_000, "total_tokens": 1_000_000}
norm = normalize_usage(usage)
total, input_cost, output_cost, native = compute_cost(
usage, norm, pricing, "embed"
)
assert native is False
assert output_cost == 0.0
assert abs(total - 0.01) < 1e-9
assert abs(input_cost - 0.01) < 1e-9
def test_app_reference_stored_in_ledger_for_chat(local_db, monkeypatch):
monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeClient_openai_gateway)
svc = GatewayService()
cfg = svc.effective_config()
cfg["gateway_force_model"] = True
cfg["gateway_model"] = "deepseek-chat"
rt = svc.runtime()
run_async(
rt.handle_chat(
{"messages": [{"role": "user", "content": "ref_test"}]},
cfg,
("guest", "ref_probe"),
"test-ua",
"my-custom-app",
)
)
row = get_table("gateway_usage_ledger").find_one(owner_id="ref_probe")
assert row is not None
assert row["app_reference"] == "my-custom-app"
assert row["backend"] == "chat"
def test_app_reference_defaults_when_not_passed(local_db, monkeypatch):
monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeClient_openai_gateway)
svc = GatewayService()
cfg = svc.effective_config()
cfg["gateway_force_model"] = True
cfg["gateway_model"] = "deepseek-chat"
rt = svc.runtime()
run_async(
rt.handle_chat(
{"messages": [{"role": "user", "content": "default_test"}]},
cfg,
("guest", "default_probe"),
"test-ua",
"default",
)
)
row = get_table("gateway_usage_ledger").find_one(owner_id="default_probe")
assert row is not None
assert row["app_reference"] == "default"
def test_app_reference_stored_in_ledger_for_embeddings(local_db, monkeypatch):
monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeEmbedClient_openai_gateway)
monkeypatch.setenv("OPENROUTER_API_KEY", "test-embed-key")
svc = GatewayService()
cfg = svc.effective_config()
cfg["gateway_embed_enabled"] = True
cfg["gateway_embed_model"] = "qwen/qwen3-embedding-8b"
rt = svc.runtime()
run_async(
rt.handle_embeddings(
{"input": "hello"},
cfg,
("guest", "embed_ref"),
"test-ua",
"devplace-embed-test-v-1-0-0",
)
)
row = get_table("gateway_usage_ledger").find_one(owner_id="embed_ref")
assert row is not None
assert row["app_reference"] == "devplace-embed-test-v-1-0-0"
assert row["backend"] == "embed"
def test_app_reference_stored_in_ledger_for_images(local_db, monkeypatch):
monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeImageClient_openai_gateway)
svc = GatewayService()
cfg = svc.effective_config()
cfg["gateway_image_enabled"] = True
cfg["gateway_image_model"] = "black-forest-labs/flux.2-pro"
rt = svc.runtime()
run_async(
rt.handle_images(
{"prompt": "ref badge"},
cfg,
("guest", "img_ref"),
"test-ua",
"devplace-image-test-v-1-0-0",
)
)
row = get_table("gateway_usage_ledger").find_one(owner_id="img_ref")
assert row is not None
assert row["app_reference"] == "devplace-image-test-v-1-0-0"
assert row["backend"] == "image"
def test_app_reference_column_exists(local_db):
db = get_table("gateway_usage_ledger").db
if "gateway_usage_ledger" not in db.tables:
return
rows = list(db.query("PRAGMA table_info('gateway_usage_ledger')"))
column_names = [r["name"] for r in rows]
assert "app_reference" in column_names
def test_chat_unknown_model_falls_back_to_default(local_db, monkeypatch):
monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeClient_openai_gateway)
svc = GatewayService()
cfg = svc.effective_config()
cfg["gateway_force_model"] = False
cfg["gateway_model"] = "deepseek-v4-flash"
rt = svc.runtime()
run_async(
rt.handle_chat(
{"model": "nonexistent-model-v99", "messages": [{"role": "user", "content": "hi"}]},
cfg,
("guest", "fallback_test_chat"),
"test",
"default",
)
)
assert rt._client.calls[-1][1]["model"] == "deepseek-v4-flash"
class FakeFallbackClient_openai_gateway:
def __init__(self, *a, **k):
self.calls = []
def build_request(self, method, url, headers=None, json=None, content=None):
return FakeRequest(method, url, json)
async def send(self, request, stream=False):
self.calls.append((request.url, request.json_body))
body = request.json_body or {}
model = body.get("model")
if model == "primary-target":
return FakeResp_openai_gateway(status=500, payload={"error": "boom"})
return FakeResp_openai_gateway(
payload={
"id": "x",
"model": model,
"choices": [{"message": {"content": "fallback ok"}}],
}
)
async def aclose(self):
pass
class FakeAlwaysFailClient_openai_gateway:
def __init__(self, *a, **k):
self.calls = []
def build_request(self, method, url, headers=None, json=None, content=None):
return FakeRequest(method, url, json)
async def send(self, request, stream=False):
self.calls.append((request.url, request.json_body))
return FakeResp_openai_gateway(status=500, payload={"error": "boom"})
async def aclose(self):
pass
class FakeBadRequestClient_openai_gateway:
def __init__(self, *a, **k):
self.calls = []
def build_request(self, method, url, headers=None, json=None, content=None):
return FakeRequest(method, url, json)
async def send(self, request, stream=False):
self.calls.append((request.url, request.json_body))
return FakeResp_openai_gateway(
status=400, payload={"error": {"message": "invalid request: bad param"}}
)
async def aclose(self):
pass
def test_chat_does_not_fall_back_on_an_unrecoverable_bad_request(local_db, monkeypatch):
from devplacepy.services.openai_gateway import routing
monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeBadRequestClient_openai_gateway)
routing.model_store.set(
routing.ModelRouteIn(source_model="fb-badreq-backup", target_model="backup-target")
)
routing.model_store.set(
routing.ModelRouteIn(
source_model="fb-badreq-primary",
target_model="primary-target",
fallback_model="fb-badreq-backup",
)
)
try:
svc = GatewayService()
cfg = svc.effective_config()
cfg["gateway_vision_enabled"] = False
cfg["gateway_max_retries"] = 0
rt = svc.runtime()
response = run_async(
rt.handle_chat(
{"model": "fb-badreq-primary", "messages": [{"role": "user", "content": "hi"}]},
cfg,
("guest", "no_fallback_on_bad_request"),
"test",
"default",
)
)
assert response.status_code == 400
assert len(rt._client.calls) == 1
finally:
routing.model_store.remove("fb-badreq-primary")
routing.model_store.remove("fb-badreq-backup")
def test_chat_falls_back_when_the_primary_model_fails(local_db, monkeypatch):
from devplacepy.services.openai_gateway import routing
monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeFallbackClient_openai_gateway)
routing.model_store.set(
routing.ModelRouteIn(source_model="fb-backup-route", target_model="backup-target")
)
routing.model_store.set(
routing.ModelRouteIn(
source_model="fb-primary-route",
target_model="primary-target",
fallback_model="fb-backup-route",
)
)
try:
svc = GatewayService()
cfg = svc.effective_config()
cfg["gateway_vision_enabled"] = False
cfg["gateway_max_retries"] = 0
rt = svc.runtime()
response = run_async(
rt.handle_chat(
{"model": "fb-primary-route", "messages": [{"role": "user", "content": "hi"}]},
cfg,
("guest", "fallback_chat_success"),
"test",
"default",
)
)
assert response.status_code == 200
assert rt._client.calls[0][1]["model"] == "primary-target"
assert rt._client.calls[-1][1]["model"] == "backup-target"
row = get_table("gateway_usage_ledger").find_one(owner_id="fallback_chat_success")
assert row is not None
assert row["requested_model"] == "fb-primary-route"
assert row["model"] == "backup-target"
assert row["success"] == 1
finally:
routing.model_store.remove("fb-primary-route")
routing.model_store.remove("fb-backup-route")
class FakeChainClient_openai_gateway:
def __init__(self, *a, **k):
self.calls = []
def build_request(self, method, url, headers=None, json=None, content=None):
return FakeRequest(method, url, json)
async def send(self, request, stream=False):
self.calls.append((request.url, request.json_body))
body = request.json_body or {}
model = body.get("model")
if model in ("chain-a-target", "chain-b-target"):
return FakeResp_openai_gateway(status=500, payload={"error": "boom"})
return FakeResp_openai_gateway(
payload={
"id": "x",
"model": model,
"choices": [{"message": {"content": "chain ok"}}],
}
)
async def aclose(self):
pass
def test_chat_walks_fallback_chain_past_failures(local_db, monkeypatch):
from devplacepy.services.openai_gateway import routing
monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeChainClient_openai_gateway)
routing.model_store.set(
routing.ModelRouteIn(source_model="chain-c", target_model="chain-c-target")
)
routing.model_store.set(
routing.ModelRouteIn(
source_model="chain-b",
target_model="chain-b-target",
fallback_model="chain-c",
)
)
routing.model_store.set(
routing.ModelRouteIn(
source_model="chain-a",
target_model="chain-a-target",
fallback_model="chain-b",
)
)
try:
svc = GatewayService()
cfg = svc.effective_config()
cfg["gateway_vision_enabled"] = False
cfg["gateway_max_retries"] = 0
rt = svc.runtime()
response = run_async(
rt.handle_chat(
{"model": "chain-a", "messages": [{"role": "user", "content": "hi"}]},
cfg,
("guest", "chain_walk"),
"test",
"default",
)
)
assert response.status_code == 200
assert [c[1]["model"] for c in rt._client.calls] == [
"chain-a-target",
"chain-b-target",
"chain-c-target",
]
row = get_table("gateway_usage_ledger").find_one(owner_id="chain_walk")
assert row["model"] == "chain-c-target"
assert row["fallback_used_route"] == "chain-c"
finally:
routing.model_store.remove("chain-a")
routing.model_store.remove("chain-b")
routing.model_store.remove("chain-c")
def test_chat_preroutes_around_unhealthy_models(local_db, monkeypatch):
from devplacepy.services.openai_gateway import health, routing
monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeClient_openai_gateway)
routing.model_store.set(
routing.ModelRouteIn(source_model="pre-good", target_model="good-target")
)
routing.model_store.set(
routing.ModelRouteIn(
source_model="pre-bad",
target_model="bad-target",
fallback_model="pre-good",
)
)
health.record_health(
"pre-bad", health.ProbeResult(False, "http_500", "boom", 1.0)
)
try:
svc = GatewayService()
cfg = svc.effective_config()
cfg["gateway_vision_enabled"] = False
rt = svc.runtime()
logged = []
response = run_async(
rt.handle_chat(
{"model": "pre-bad", "messages": [{"role": "user", "content": "hi"}]},
cfg,
("guest", "preroute"),
"test",
"default",
log=logged.append,
)
)
assert response.status_code == 200
assert len(rt._client.calls) == 1
assert rt._client.calls[0][1]["model"] == "good-target"
assert any("is unhealthy, starting on" in line for line in logged)
row = get_table("gateway_usage_ledger").find_one(owner_id="preroute")
assert row["requested_model"] == "pre-bad"
assert row["model"] == "good-target"
finally:
routing.model_store.remove("pre-bad")
routing.model_store.remove("pre-good")
def test_chat_tries_original_when_chain_is_exhausted(local_db, monkeypatch):
from devplacepy.services.openai_gateway import health, routing
monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeClient_openai_gateway)
routing.model_store.set(
routing.ModelRouteIn(
source_model="exh-fb",
target_model="exh-fb-target",
fallback_model="exh-primary",
)
)
routing.model_store.set(
routing.ModelRouteIn(
source_model="exh-primary",
target_model="exh-primary-target",
fallback_model="exh-fb",
)
)
health.record_health(
"exh-primary", health.ProbeResult(False, "http_500", "boom", 1.0)
)
health.record_health(
"exh-fb", health.ProbeResult(False, "http_500", "boom", 1.0)
)
try:
svc = GatewayService()
cfg = svc.effective_config()
cfg["gateway_vision_enabled"] = False
rt = svc.runtime()
response = run_async(
rt.handle_chat(
{"model": "exh-primary", "messages": [{"role": "user", "content": "hi"}]},
cfg,
("guest", "exhausted"),
"test",
"default",
)
)
assert response.status_code == 200
assert [c[1]["model"] for c in rt._client.calls] == ["exh-primary-target"]
finally:
routing.model_store.remove("exh-primary")
routing.model_store.remove("exh-fb")
def test_models_endpoint_hides_unhealthy_routes(local_db):
from devplacepy.services.openai_gateway import health, routing
routing.model_store.set(
routing.ModelRouteIn(source_model="hide-me", target_model="v/hide")
)
health.record_health(
"hide-me", health.ProbeResult(False, "http_500", "boom", 1.0)
)
try:
svc = GatewayService()
payload = json.loads(svc._models_response().body.decode())
assert "hide-me" not in {m["id"] for m in payload["data"]}
health.record_health(
"hide-me", health.ProbeResult(True, "ok", "", 1.0)
)
payload = json.loads(svc._models_response().body.decode())
assert "hide-me" in {m["id"] for m in payload["data"]}
finally:
routing.model_store.remove("hide-me")
class FakeContextLengthClient_openai_gateway:
def __init__(self, *a, **k):
self.calls = []
def build_request(self, method, url, headers=None, json=None, content=None):
return FakeRequest(method, url, json)
async def send(self, request, stream=False):
self.calls.append((request.url, request.json_body))
body = request.json_body or {}
model = body.get("model")
if model == "ctx-primary-target":
return FakeResp_openai_gateway(
status=400,
payload={"error": {"message": "maximum context length exceeded"}},
)
return FakeResp_openai_gateway(
payload={
"id": "x",
"model": model,
"choices": [{"message": {"content": "fallback ok"}}],
}
)
async def aclose(self):
pass
def test_chat_falls_back_on_a_context_length_error(local_db, monkeypatch):
from devplacepy.services.openai_gateway import routing
monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeContextLengthClient_openai_gateway)
routing.model_store.set(
routing.ModelRouteIn(source_model="ctx-backup-route", target_model="ctx-backup-target")
)
routing.model_store.set(
routing.ModelRouteIn(
source_model="ctx-primary-route",
target_model="ctx-primary-target",
fallback_model="ctx-backup-route",
)
)
try:
svc = GatewayService()
cfg = svc.effective_config()
cfg["gateway_vision_enabled"] = False
cfg["gateway_max_retries"] = 0
rt = svc.runtime()
response = run_async(
rt.handle_chat(
{"model": "ctx-primary-route", "messages": [{"role": "user", "content": "hi"}]},
cfg,
("guest", "context_length_fallback_success"),
"test",
"default",
)
)
assert response.status_code == 200
assert rt._client.calls[0][1]["model"] == "ctx-primary-target"
assert rt._client.calls[-1][1]["model"] == "ctx-backup-target"
row = get_table("gateway_usage_ledger").find_one(
owner_id="context_length_fallback_success"
)
assert row["success"] == 1
assert row["fallback_used_route"] == "ctx-backup-route"
finally:
routing.model_store.remove("ctx-primary-route")
routing.model_store.remove("ctx-backup-route")
def test_chat_skips_a_too_small_primary_and_goes_straight_to_fallback(local_db, monkeypatch):
from devplacepy.services.openai_gateway import routing
monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeContextLengthClient_openai_gateway)
routing.model_store.set(
routing.ModelRouteIn(
source_model="ctx-precheck-backup",
target_model="ctx-backup-target",
context_window=1_000_000,
)
)
routing.model_store.set(
routing.ModelRouteIn(
source_model="ctx-precheck-primary",
target_model="ctx-primary-target",
context_window=50,
fallback_model="ctx-precheck-backup",
)
)
try:
svc = GatewayService()
cfg = svc.effective_config()
cfg["gateway_vision_enabled"] = False
cfg["gateway_max_retries"] = 0
rt = svc.runtime()
long_message = "word " * 2000
response = run_async(
rt.handle_chat(
{
"model": "ctx-precheck-primary",
"messages": [{"role": "user", "content": long_message}],
},
cfg,
("guest", "context_precheck_skips_primary"),
"test",
"default",
)
)
assert response.status_code == 200
# Only the fallback was ever called - the primary was known too small.
assert len(rt._client.calls) == 1
assert rt._client.calls[0][1]["model"] == "ctx-backup-target"
row = get_table("gateway_usage_ledger").find_one(
owner_id="context_precheck_skips_primary"
)
assert row["fallback_used_route"] == "ctx-precheck-backup"
finally:
routing.model_store.remove("ctx-precheck-primary")
routing.model_store.remove("ctx-precheck-backup")
def test_chat_falls_back_via_molodetz_when_the_client_sends_an_unrouted_model_name(
local_db, monkeypatch
):
from devplacepy.services.openai_gateway import routing
original = routing.model_store.get("molodetz")
routing.model_store.set(
routing.ModelRouteIn(source_model="fb-molodetz-backup", target_model="backup-target")
)
routing.model_store.set(
routing.ModelRouteIn(
source_model="molodetz",
target_model="primary-target",
fallback_model="fb-molodetz-backup",
)
)
try:
svc = GatewayService()
cfg = svc.effective_config()
cfg["gateway_vision_enabled"] = False
cfg["gateway_max_retries"] = 0
cfg["gateway_force_model"] = False
base_default_model = cfg["gateway_model"]
class FakeUnroutedPrimaryFailsClient:
def __init__(self, *a, **k):
self.calls = []
def build_request(self, method, url, headers=None, json=None, content=None):
return FakeRequest(method, url, json)
async def send(self, request, stream=False):
self.calls.append((request.url, request.json_body))
body = request.json_body or {}
model = body.get("model")
if model == base_default_model:
return FakeResp_openai_gateway(
status=402, payload={"error": {"message": "Insufficient Balance"}}
)
return FakeResp_openai_gateway(
payload={
"id": "x",
"model": model,
"choices": [{"message": {"content": "fallback ok"}}],
}
)
async def aclose(self):
pass
monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeUnroutedPrimaryFailsClient)
rt = svc.runtime()
# The client sends a model name that matches no configured route at all
# (as a caller like dpc does, which is never told the exact "molodetz"
# routing alias) - the primary call falls through to the flat/global
# default model (no route overlay applies), but a failure there must
# still consult molodetz's own configured fallback, not silently skip
# it just because the client's literal string never matched anything.
response = run_async(
rt.handle_chat(
{
"model": "some-client-side-model-name",
"messages": [{"role": "user", "content": "hi"}],
},
cfg,
("guest", "fallback_chat_unrouted_client_model"),
"test",
"default",
)
)
assert response.status_code == 200
assert rt._client.calls[0][1]["model"] == base_default_model
assert rt._client.calls[-1][1]["model"] == "backup-target"
row = get_table("gateway_usage_ledger").find_one(
owner_id="fallback_chat_unrouted_client_model"
)
assert row is not None
assert row["requested_model"] == "some-client-side-model-name"
assert row["model"] == "backup-target"
assert row["success"] == 1
finally:
routing.model_store.remove("fb-molodetz-backup")
if original is not None:
routing.model_store.set(
routing.ModelRouteIn(
**{
field: getattr(original, field)
for field in routing.ModelRouteIn.model_fields
}
)
)
else:
routing.model_store.remove("molodetz")
def test_chat_returns_the_fallback_failure_when_both_models_fail(local_db, monkeypatch):
from devplacepy.services.openai_gateway import routing
monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeAlwaysFailClient_openai_gateway)
routing.model_store.set(
routing.ModelRouteIn(source_model="fb-backup-route2", target_model="backup-target2")
)
routing.model_store.set(
routing.ModelRouteIn(
source_model="fb-primary-route2",
target_model="primary-target2",
fallback_model="fb-backup-route2",
)
)
try:
svc = GatewayService()
cfg = svc.effective_config()
cfg["gateway_vision_enabled"] = False
cfg["gateway_max_retries"] = 0
rt = svc.runtime()
response = run_async(
rt.handle_chat(
{"model": "fb-primary-route2", "messages": [{"role": "user", "content": "hi"}]},
cfg,
("guest", "fallback_chat_both_fail"),
"test",
"default",
)
)
assert response.status_code == 500
assert len(rt._client.calls) == 2
assert rt._client.calls[0][1]["model"] == "primary-target2"
assert rt._client.calls[1][1]["model"] == "backup-target2"
finally:
routing.model_store.remove("fb-primary-route2")
routing.model_store.remove("fb-backup-route2")
def test_chat_without_a_fallback_never_makes_a_second_call(local_db, monkeypatch):
from devplacepy.services.openai_gateway import routing
monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeAlwaysFailClient_openai_gateway)
routing.model_store.set(
routing.ModelRouteIn(source_model="fb-no-fallback", target_model="no-fallback-target")
)
try:
svc = GatewayService()
cfg = svc.effective_config()
cfg["gateway_vision_enabled"] = False
cfg["gateway_max_retries"] = 0
rt = svc.runtime()
run_async(
rt.handle_chat(
{"model": "fb-no-fallback", "messages": [{"role": "user", "content": "hi"}]},
cfg,
("guest", "no_fallback_configured"),
"test",
"default",
)
)
assert len(rt._client.calls) == 1
finally:
routing.model_store.remove("fb-no-fallback")
def test_embeddings_unknown_model_falls_back_to_default(local_db, monkeypatch):
monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeEmbedClient_openai_gateway)
monkeypatch.setenv("OPENROUTER_API_KEY", "test-embed-key")
svc = GatewayService()
cfg = svc.effective_config()
cfg["gateway_force_model"] = False
cfg["gateway_embed_enabled"] = True
cfg["gateway_embed_model"] = "qwen/qwen3-embedding-8b"
rt = svc.runtime()
run_async(
rt.handle_embeddings(
{"model": "nonexistent-embed-model", "input": "hello"},
cfg,
("guest", "fallback_test_embed"),
"test",
"default",
)
)
assert rt._client.calls[-1][1]["model"] == "qwen/qwen3-embedding-8b"
def test_images_unknown_model_falls_back_to_default(local_db, monkeypatch):
monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeImageClient_openai_gateway)
svc = GatewayService()
cfg = svc.effective_config()
cfg["gateway_force_model"] = False
cfg["gateway_image_enabled"] = True
cfg["gateway_image_model"] = "black-forest-labs/flux.2-pro"
rt = svc.runtime()
run_async(
rt.handle_images(
{"model": "nonexistent-image-model", "prompt": "test badge"},
cfg,
("guest", "fallback_test_img"),
"test",
"default",
)
)
assert rt._client.calls[-1][1]["model"] == "black-forest-labs/flux.2-pro"
class FakeClientWithUsage_openai_gateway(FakeClient_openai_gateway):
async def send(self, request, stream=False):
self.calls.append((request.url, request.json_body))
body = request.json_body or {}
return FakeResp_openai_gateway(
payload={
"id": "x",
"model": body.get("model"),
"choices": [{"message": {"content": "hi there"}}],
"usage": {"prompt_tokens": 7, "completion_tokens": 3, "total_tokens": 10},
}
)
def test_ollama_stream_does_not_send_stream_options(local_db, monkeypatch):
monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeClient_openai_gateway)
svc = GatewayService()
cfg = svc.effective_config()
cfg["gateway_upstream_url"] = "http://127.0.0.1:11434/api/chat"
rt = svc.runtime()
run_async(
rt.handle_chat(
{
"messages": [{"role": "user", "content": "hi"}],
"stream": True,
"stream_options": {"include_usage": True},
},
cfg,
("guest", "ollama_stream_opts"),
"test",
"default",
)
)
upstream_payload = rt._client.calls[-1][1]
assert upstream_payload["stream"] is True
assert "stream_options" not in upstream_payload
assert upstream_payload["think"] is False
def test_ollama_explicit_dialect_overrides_url(local_db, monkeypatch):
monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeClient_openai_gateway)
svc = GatewayService()
cfg = svc.effective_config()
cfg["gateway_upstream_url"] = "https://ai.example.com/v1/chat/completions"
cfg["gateway_thinking_dialect"] = "ollama"
rt = svc.runtime()
run_async(
rt.handle_chat(
{"messages": [{"role": "user", "content": "hi"}]},
cfg,
("guest", "ollama_dialect_override"),
"test",
"default",
)
)
body = rt._client.calls[-1][1]
assert body["think"] is False
assert "thinking" not in body
def test_client_model_allowed_when_allow_client_model_on(local_db, monkeypatch):
monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeClient_openai_gateway)
svc = GatewayService()
cfg = svc.effective_config()
cfg["gateway_force_model"] = True
cfg["gateway_allow_client_model"] = True
cfg["gateway_model"] = "deepseek-chat"
rt = svc.runtime()
run_async(
rt.handle_chat(
{"model": "llama3.2", "messages": [{"role": "user", "content": "hi"}]},
cfg,
("guest", "client_model"),
"test",
"default",
)
)
assert rt._client.calls[-1][1]["model"] == "llama3.2"
def test_molodetz_alias_still_remapped_with_allow_client_model(local_db, monkeypatch):
from devplacepy.services.openai_gateway import routing
monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeClient_openai_gateway)
original = routing.model_store.get("molodetz")
routing.model_store.remove("molodetz")
try:
svc = GatewayService()
cfg = svc.effective_config()
cfg["gateway_force_model"] = True
cfg["gateway_allow_client_model"] = True
cfg["gateway_model"] = "deepseek-chat"
rt = svc.runtime()
run_async(
rt.handle_chat(
{"model": "molodetz", "messages": [{"role": "user", "content": "hi"}]},
cfg,
("guest", "molodetz_alias"),
"test",
"default",
)
)
assert rt._client.calls[-1][1]["model"] == "deepseek-chat"
finally:
if original is not None:
routing.model_store.set(
routing.ModelRouteIn(
**{
field: getattr(original, field)
for field in routing.ModelRouteIn.model_fields
}
)
)
def test_stream_forwarded_to_upstream_with_forced_include_usage(local_db, monkeypatch):
monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeClient_openai_gateway)
svc = GatewayService()
cfg = svc.effective_config()
rt = svc.runtime()
run_async(
rt.handle_chat(
{
"messages": [{"role": "user", "content": "hi"}],
"stream": True,
"stream_options": {"include_usage": False},
},
cfg,
("guest", "test"),
"test",
"default",
)
)
upstream_payload = rt._client.calls[-1][1]
assert upstream_payload["stream"] is True
assert upstream_payload["stream_options"] == {"include_usage": True}
assert upstream_payload["thinking"] == {"type": "disabled"}
assert "think" not in upstream_payload
assert "reasoning" not in upstream_payload
def test_non_stream_has_no_stream_options_upstream(local_db, monkeypatch):
monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeClient_openai_gateway)
svc = GatewayService()
cfg = svc.effective_config()
rt = svc.runtime()
run_async(
rt.handle_chat(
{
"messages": [{"role": "user", "content": "hi"}],
"stream_options": {"include_usage": True},
},
cfg,
("guest", "test"),
"test",
"default",
)
)
upstream_payload = rt._client.calls[-1][1]
assert "stream_options" not in upstream_payload
assert upstream_payload["stream"] is False
def test_thinking_disabled_by_default(local_db, monkeypatch):
monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeClient_openai_gateway)
svc = GatewayService()
cfg = svc.effective_config()
rt = svc.runtime()
run_async(
rt.handle_chat(
{"messages": [{"role": "user", "content": "hi"}]},
cfg,
("guest", "test"),
"test",
"default",
)
)
body = rt._client.calls[-1][1]
assert body["thinking"] == {"type": "disabled"}
def test_client_think_true_enables_upstream_thinking(local_db, monkeypatch):
monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeClient_openai_gateway)
svc = GatewayService()
cfg = svc.effective_config()
rt = svc.runtime()
run_async(
rt.handle_chat(
{
"messages": [{"role": "user", "content": "hi"}],
"think": True,
},
cfg,
("guest", "test"),
"test",
"default",
)
)
body = rt._client.calls[-1][1]
assert body["thinking"] == {"type": "enabled"}
assert "think" not in body
def test_openrouter_upstream_gets_reasoning_none(local_db, monkeypatch):
monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeClient_openai_gateway)
svc = GatewayService()
cfg = svc.effective_config()
cfg["gateway_upstream_url"] = "https://openrouter.ai/api/v1/chat/completions"
rt = svc.runtime()
run_async(
rt.handle_chat(
{"messages": [{"role": "user", "content": "hi"}]},
cfg,
("guest", "test"),
"test",
"default",
)
)
body = rt._client.calls[-1][1]
assert body["reasoning"] == {"effort": "none"}
assert "thinking" not in body
def test_include_usage_emits_usage_chunk(local_db, monkeypatch):
monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeClientWithUsage_openai_gateway)
svc = GatewayService()
cfg = svc.effective_config()
rt = svc.runtime()
resp = run_async(
rt.handle_chat(
{
"messages": [{"role": "user", "content": "hi"}],
"stream": True,
"stream_options": {"include_usage": True},
},
cfg,
("guest", "test"),
"test",
"default",
)
)
async def drain():
out = []
async for chunk in resp.body_iterator:
out.append(chunk if isinstance(chunk, str) else chunk.decode())
return "".join(out)
body = run_async(drain())
assert '"usage"' in body
assert '"total_tokens": 10' in body
assert "[DONE]" in body
def test_no_usage_chunk_without_include_usage(local_db, monkeypatch):
monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeClientWithUsage_openai_gateway)
svc = GatewayService()
cfg = svc.effective_config()
rt = svc.runtime()
resp = run_async(
rt.handle_chat(
{"messages": [{"role": "user", "content": "hi"}], "stream": True},
cfg,
("guest", "test"),
"test",
"default",
)
)
async def drain():
out = []
async for chunk in resp.body_iterator:
out.append(chunk if isinstance(chunk, str) else chunk.decode())
return "".join(out)
body = run_async(drain())
assert '"usage"' not in body
assert "[DONE]" in body
def test_stream_headers_carry_no_cost_or_token_data(local_db, monkeypatch):
monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeClientWithUsage_openai_gateway)
svc = GatewayService()
cfg = svc.effective_config()
rt = svc.runtime()
resp = run_async(
rt.handle_chat(
{"messages": [{"role": "user", "content": "hi"}], "stream": True},
cfg,
("guest", "test"),
"test",
"my-app",
)
)
assert resp.headers["X-Gateway-Model"] == cfg["gateway_model"]
assert resp.headers["X-Gateway-Backend"] == "chat"
assert resp.headers["X-App-Reference"] == "my-app"
assert "x-gateway-cost-usd" not in {k.lower() for k in resp.headers}
assert "x-gateway-total-tokens" not in {k.lower() for k in resp.headers}
def test_stream_records_ttft_and_inter_token_in_ledger(local_db, monkeypatch):
monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeClientWithUsage_openai_gateway)
svc = GatewayService()
cfg = svc.effective_config()
rt = svc.runtime()
resp = run_async(
rt.handle_chat(
{
"messages": [{"role": "user", "content": "hi"}],
"stream": True,
"stream_options": {"include_usage": True},
},
cfg,
("guest", "ttft_probe"),
"test",
"default",
)
)
async def drain():
async for _ in resp.body_iterator:
pass
run_async(drain())
row = get_table("gateway_usage_ledger").find_one(owner_id="ttft_probe")
assert row is not None
assert row["backend"] == "chat"
assert row["stream_requested"] == 1
assert row["success"] == 1
assert row["total_tokens"] == 10
assert row["ttft_ms"] is not None and row["ttft_ms"] >= 0.0
assert row["inter_token_ms"] is not None and row["inter_token_ms"] >= 0.0
class FakeClientReasoning_openai_gateway(FakeClient_openai_gateway):
async def send(self, request, stream=False):
self.calls.append((request.url, request.json_body))
body = request.json_body or {}
return FakeResp_openai_gateway(
payload={
"id": "x",
"model": body.get("model"),
"choices": [{"message": {"content": "x", "reasoning": "aaaaa"}}],
"usage": {"prompt_tokens": 7, "completion_tokens": 3, "total_tokens": 10},
}
)
def test_ollama_reasoning_delta_is_counted_as_a_content_chunk(local_db, monkeypatch):
monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeClientReasoning_openai_gateway)
svc = GatewayService()
cfg = svc.effective_config()
rt = svc.runtime()
resp = run_async(
rt.handle_chat(
{
"messages": [{"role": "user", "content": "hi"}],
"stream": True,
"stream_options": {"include_usage": True},
},
cfg,
("guest", "ollama_reasoning_probe"),
"test",
"default",
)
)
async def drain():
async for _ in resp.body_iterator:
pass
run_async(drain())
row = get_table("gateway_usage_ledger").find_one(owner_id="ollama_reasoning_probe")
assert row is not None
assert row["inter_token_ms"] is not None and row["inter_token_ms"] >= 0.0
def test_stream_client_disconnect_records_failure_and_closes_upstream(local_db, monkeypatch):
monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeClient_openai_gateway)
svc = GatewayService()
cfg = svc.effective_config()
rt = svc.runtime()
resp = run_async(
rt.handle_chat(
{"messages": [{"role": "user", "content": "hi"}], "stream": True},
cfg,
("guest", "disconnect_probe"),
"test",
"default",
)
)
async def partial_then_close():
agen = resp.body_iterator
await agen.__anext__()
await agen.aclose()
run_async(partial_then_close())
row = get_table("gateway_usage_ledger").find_one(owner_id="disconnect_probe")
assert row is not None
assert row["success"] == 0
assert row["error_category"] == "client_disconnected"
def test_models_endpoint_publishes_molodetz(local_db):
from devplacepy.services.openai_gateway import routing
routing.seed_default_deepseek_routes()
try:
svc = GatewayService()
resp = svc._models_response()
payload = json.loads(resp.body.decode())
ids = {m["id"] for m in payload["data"]}
assert payload["object"] == "list"
assert {"molodetz", "molodetz-pro"} <= ids
for model in payload["data"]:
assert model["object"] == "model"
assert model["owned_by"] == "molodetz"
finally:
routing.model_store.remove("molodetz")
routing.model_store.remove("molodetz-pro")
# Upstream header collision policy: only X-Gateway-Model is ever trusted and
# forwarded from an upstream that happens to speak our own X-Gateway-* header
# convention (e.g. another DevPlace-style gateway). Every other figure
# (cost/tokens/latency/context/app-reference) must remain ours regardless of
# what an upstream claims. See usage.upstream_reported_model / CLAUDE.md.
class FakeClientUpstreamModelHeader_openai_gateway(FakeClient_openai_gateway):
async def send(self, request, stream=False):
self.calls.append((request.url, request.json_body))
body = request.json_body or {}
return FakeResp_openai_gateway(
payload={
"id": "x",
"model": body.get("model"),
"choices": [{"message": {"content": "hi there"}}],
},
extra_headers={"X-Gateway-Model": "upstream/served-model-x"},
)
class FakeClientHostileModelHeader_openai_gateway(FakeClient_openai_gateway):
async def send(self, request, stream=False):
self.calls.append((request.url, request.json_body))
body = request.json_body or {}
return FakeResp_openai_gateway(
payload={
"id": "x",
"model": body.get("model"),
"choices": [{"message": {"content": "hi there"}}],
},
extra_headers={"X-Gateway-Model": "evil\r\nX-Injected: true"},
)
class FakeEmbedClientUpstreamModelHeader_openai_gateway(FakeEmbedClient_openai_gateway):
async def send(self, request, stream=False):
self.calls.append((request.url, request.json_body))
body = request.json_body or {}
return FakeResp_openai_gateway(
payload={
"object": "list",
"model": body.get("model"),
"data": [{"object": "embedding", "index": 0, "embedding": [0.1, 0.2]}],
"usage": {"prompt_tokens": 5, "total_tokens": 5},
},
extra_headers={"X-Gateway-Model": "upstream/embed-served"},
)
class FakeImageClientUpstreamModelHeader_openai_gateway(FakeImageClient_openai_gateway):
async def send(self, request, stream=False):
self.calls.append((request.url, request.json_body))
body = request.json_body or {}
return FakeResp_openai_gateway(
payload={
"created": 1,
"model": body.get("model"),
"data": [{"b64_json": "aGVsbG8="}],
"usage": {"cost": 0.05},
},
extra_headers={"X-Gateway-Model": "upstream/image-served"},
)
class FakePassthroughClientUpstreamModelHeader_openai_gateway:
def __init__(self, *a, **k):
self.calls = []
def build_request(self, method, url, headers=None, json=None, content=None):
return FakeRequest(method, url, json)
async def send(self, request, stream=False):
self.calls.append((request.url, request.json_body))
return FakeResp_openai_gateway(
payload={"ok": True},
extra_headers={"X-Gateway-Model": "upstream/passthrough-served"},
)
async def aclose(self):
pass
def test_upstream_x_gateway_model_header_is_forwarded_for_chat(local_db, monkeypatch):
monkeypatch.setattr(
gwmod.httpx, "AsyncClient", FakeClientUpstreamModelHeader_openai_gateway
)
svc = GatewayService()
cfg = svc.effective_config()
cfg["gateway_force_model"] = True
cfg["gateway_model"] = "deepseek-chat"
rt = svc.runtime()
resp = run_async(
rt.handle_chat(
{"messages": [{"role": "user", "content": "hi"}]},
cfg,
("guest", "upstream_model_chat"),
"test",
"default",
)
)
assert resp.headers["X-Gateway-Model"] == "upstream/served-model-x"
row = get_table("gateway_usage_ledger").find_one(owner_id="upstream_model_chat")
assert row is not None
assert row["model"] == "deepseek-chat"
def test_upstream_x_gateway_model_header_is_forwarded_for_streaming_chat(
local_db, monkeypatch
):
monkeypatch.setattr(
gwmod.httpx, "AsyncClient", FakeClientUpstreamModelHeader_openai_gateway
)
svc = GatewayService()
cfg = svc.effective_config()
rt = svc.runtime()
resp = run_async(
rt.handle_chat(
{"messages": [{"role": "user", "content": "hi"}], "stream": True},
cfg,
("guest", "upstream_model_stream"),
"test",
"default",
)
)
assert resp.headers["X-Gateway-Model"] == "upstream/served-model-x"
assert resp.headers["X-Gateway-Backend"] == "chat"
async def drain():
async for _ in resp.body_iterator:
pass
run_async(drain())
def test_upstream_x_gateway_model_header_is_forwarded_for_embeddings(
local_db, monkeypatch
):
monkeypatch.setattr(
gwmod.httpx, "AsyncClient", FakeEmbedClientUpstreamModelHeader_openai_gateway
)
monkeypatch.setenv("OPENROUTER_API_KEY", "test-embed-key")
svc = GatewayService()
cfg = svc.effective_config()
cfg["gateway_embed_enabled"] = True
cfg["gateway_embed_model"] = "qwen/qwen3-embedding-8b"
rt = svc.runtime()
resp = run_async(
rt.handle_embeddings(
{"input": "hello"},
cfg,
("guest", "upstream_model_embed"),
"test",
"default",
)
)
assert resp.headers["X-Gateway-Model"] == "upstream/embed-served"
row = get_table("gateway_usage_ledger").find_one(owner_id="upstream_model_embed")
assert row is not None
assert row["model"] == "qwen/qwen3-embedding-8b"
def test_upstream_x_gateway_model_header_is_forwarded_for_images(local_db, monkeypatch):
monkeypatch.setattr(
gwmod.httpx, "AsyncClient", FakeImageClientUpstreamModelHeader_openai_gateway
)
svc = GatewayService()
cfg = svc.effective_config()
cfg["gateway_image_enabled"] = True
cfg["gateway_image_model"] = "black-forest-labs/flux.2-pro"
rt = svc.runtime()
resp = run_async(
rt.handle_images(
{"prompt": "badge"},
cfg,
("guest", "upstream_model_img"),
"test",
"default",
)
)
assert resp.headers["X-Gateway-Model"] == "upstream/image-served"
row = get_table("gateway_usage_ledger").find_one(owner_id="upstream_model_img")
assert row is not None
assert row["model"] == "black-forest-labs/flux.2-pro"
def test_upstream_x_gateway_model_header_is_forwarded_for_passthrough(
local_db, monkeypatch
):
monkeypatch.setattr(
gwmod.httpx, "AsyncClient", FakePassthroughClientUpstreamModelHeader_openai_gateway
)
svc = GatewayService()
cfg = svc.effective_config()
cfg["gateway_model"] = "deepseek-chat"
rt = svc.runtime()
resp = run_async(
rt.handle_passthrough(
"POST",
"responses",
"application/json",
b"{}",
cfg,
("guest", "upstream_model_passthrough"),
"test",
"default",
)
)
assert resp.headers["X-Gateway-Model"] == "upstream/passthrough-served"
row = get_table("gateway_usage_ledger").find_one(
owner_id="upstream_model_passthrough"
)
assert row is not None
assert row["model"] == "deepseek-chat"
def test_no_upstream_model_header_keeps_our_own_model(local_db, monkeypatch):
monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeClient_openai_gateway)
svc = GatewayService()
cfg = svc.effective_config()
cfg["gateway_force_model"] = True
cfg["gateway_model"] = "deepseek-chat"
rt = svc.runtime()
resp = run_async(
rt.handle_chat(
{"messages": [{"role": "user", "content": "hi"}]},
cfg,
("guest", "no_upstream_header"),
"test",
"default",
)
)
assert resp.headers["X-Gateway-Model"] == "deepseek-chat"
class FakeEmbeddedErrorClient_openai_gateway:
"""Mimics OpenRouter's documented behavior of answering 200 OK with the
failure embedded in the JSON body (openrouter.ai/docs/api-reference/errors)
instead of a non-2xx status."""
def __init__(self, *a, **k):
self.calls = []
def build_request(self, method, url, headers=None, json=None, content=None):
return FakeRequest(method, url, json)
async def send(self, request, stream=False):
self.calls.append((request.url, request.json_body))
body = request.json_body or {}
model = body.get("model")
if model == "primary-target":
return FakeResp_openai_gateway(
status=200,
payload={"error": {"message": "no provider available", "code": 502}},
)
return FakeResp_openai_gateway(
payload={
"id": "x",
"model": model,
"choices": [{"message": {"content": "fallback ok"}}],
}
)
async def aclose(self):
pass
def test_chat_falls_back_when_upstream_returns_200_with_an_embedded_error(
local_db, monkeypatch
):
from devplacepy.services.openai_gateway import routing
monkeypatch.setattr(
gwmod.httpx, "AsyncClient", FakeEmbeddedErrorClient_openai_gateway
)
routing.model_store.set(
routing.ModelRouteIn(source_model="eb-backup-route", target_model="backup-target")
)
routing.model_store.set(
routing.ModelRouteIn(
source_model="eb-primary-route",
target_model="primary-target",
fallback_model="eb-backup-route",
)
)
try:
svc = GatewayService()
cfg = svc.effective_config()
cfg["gateway_vision_enabled"] = False
cfg["gateway_max_retries"] = 0
rt = svc.runtime()
response = run_async(
rt.handle_chat(
{"model": "eb-primary-route", "messages": [{"role": "user", "content": "hi"}]},
cfg,
("guest", "embedded_error_chat"),
"test",
"default",
)
)
assert response.status_code == 200
assert rt._client.calls[0][1]["model"] == "primary-target"
assert rt._client.calls[-1][1]["model"] == "backup-target"
row = get_table("gateway_usage_ledger").find_one(owner_id="embedded_error_chat")
assert row is not None
assert row["requested_model"] == "eb-primary-route"
assert row["model"] == "backup-target"
assert row["success"] == 1
finally:
routing.model_store.remove("eb-primary-route")
routing.model_store.remove("eb-backup-route")
class FakeAlwaysEmbeddedErrorClient_openai_gateway:
def __init__(self, *a, **k):
self.calls = []
def build_request(self, method, url, headers=None, json=None, content=None):
return FakeRequest(method, url, json)
async def send(self, request, stream=False):
self.calls.append((request.url, request.json_body))
return FakeResp_openai_gateway(
status=200,
payload={"error": {"message": "no provider available", "code": 502}},
)
async def aclose(self):
pass
def test_chat_reports_502_when_upstream_returns_200_with_an_embedded_error_and_no_fallback(
local_db, monkeypatch
):
monkeypatch.setattr(
gwmod.httpx, "AsyncClient", FakeAlwaysEmbeddedErrorClient_openai_gateway
)
svc = GatewayService()
cfg = svc.effective_config()
cfg["gateway_vision_enabled"] = False
cfg["gateway_max_retries"] = 0
rt = svc.runtime()
response = run_async(
rt.handle_chat(
{"messages": [{"role": "user", "content": "hi"}]},
cfg,
("guest", "embedded_error_no_fallback"),
"test",
"default",
)
)
assert response.status_code == 502
row = get_table("gateway_usage_ledger").find_one(owner_id="embedded_error_no_fallback")
assert row is not None
assert row["success"] == 0
assert row["error_category"] == "upstream_error"
def test_ollama_dialect_sends_reasoning_effort_alongside_think(local_db, monkeypatch):
# Ollama's native /api/chat honors a boolean `think`, but its
# OpenAI-compatible /v1/chat/completions layer (what this gateway
# actually calls) does not - it maps reasoning_effort/reasoning
# instead (github.com/ollama/ollama issues #15288, #15293, #14820).
monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeClient_openai_gateway)
svc = GatewayService()
cfg = svc.effective_config()
cfg["gateway_upstream_url"] = "http://127.0.0.1:11434/v1/chat/completions"
rt = svc.runtime()
run_async(
rt.handle_chat(
{"messages": [{"role": "user", "content": "hi"}]},
cfg,
("guest", "ollama_reasoning_effort"),
"test",
"default",
)
)
body = rt._client.calls[-1][1]
assert body["think"] is False
assert body["reasoning_effort"] == "none"
def test_hostile_upstream_model_header_is_ignored_end_to_end(local_db, monkeypatch):
monkeypatch.setattr(
gwmod.httpx, "AsyncClient", FakeClientHostileModelHeader_openai_gateway
)
svc = GatewayService()
cfg = svc.effective_config()
cfg["gateway_force_model"] = True
cfg["gateway_model"] = "deepseek-chat"
rt = svc.runtime()
resp = run_async(
rt.handle_chat(
{"messages": [{"role": "user", "content": "hi"}]},
cfg,
("guest", "hostile_header"),
"test",
"default",
)
)
assert resp.headers["X-Gateway-Model"] == "deepseek-chat"
def test_bypass_preamble_skips_the_operator_preamble_when_allowed(local_db, monkeypatch):
monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeClient_openai_gateway)
svc = GatewayService()
cfg = svc.effective_config()
cfg["gateway_system_preamble"] = "OPERATOR-PREAMBLE-TEXT"
rt = svc.runtime()
run_async(
rt.handle_chat(
{
"messages": [
{"role": "system", "content": "Reply with strict JSON only."},
{"role": "user", "content": "hi"},
],
"bypass_preamble": True,
},
cfg,
("internal", "devii"),
"test",
"default",
bypass_allowed=True,
)
)
sent = rt._client.calls[-1][1]
system_message = sent["messages"][0]
assert system_message["role"] == "system"
assert "OPERATOR-PREAMBLE-TEXT" not in system_message["content"]
assert "Current date" in system_message["content"]
assert "Reply with strict JSON only." in system_message["content"]
assert "bypass_preamble" not in sent
def test_bypass_preamble_ignored_without_internal_credentials(local_db, monkeypatch):
monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeClient_openai_gateway)
svc = GatewayService()
cfg = svc.effective_config()
cfg["gateway_system_preamble"] = "OPERATOR-PREAMBLE-TEXT"
rt = svc.runtime()
run_async(
rt.handle_chat(
{"messages": [{"role": "user", "content": "hi"}], "bypass_preamble": True},
cfg,
("user", "someuser"),
"test",
"default",
bypass_allowed=False,
)
)
sent = rt._client.calls[-1][1]
system_message = sent["messages"][0]
assert "OPERATOR-PREAMBLE-TEXT" in system_message["content"]
assert "bypass_preamble" not in sent
def test_bypass_preamble_keeps_the_clients_own_system_content(local_db, monkeypatch):
monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeClient_openai_gateway)
svc = GatewayService()
cfg = svc.effective_config()
cfg["gateway_system_preamble"] = "OPERATOR-PREAMBLE-TEXT"
rt = svc.runtime()
run_async(
rt.handle_chat(
{
"messages": [
{"role": "system", "content": "Grade strictly. Reply JSON only."},
{"role": "user", "content": "hi"},
],
"bypass_preamble": True,
},
cfg,
("internal", "devii"),
"test",
"default",
bypass_allowed=True,
)
)
sent = rt._client.calls[-1][1]
system_message = sent["messages"][0]
assert "OPERATOR-PREAMBLE-TEXT" not in system_message["content"]
assert "Grade strictly. Reply JSON only." in system_message["content"]
def test_internal_bypass_allowed_requires_the_exact_internal_key(local_db):
set_setting("gateway_internal_key", "test-internal-secret")
try:
svc = GatewayService()
cfg = svc.effective_config()
assert (
svc.internal_bypass_allowed(
_make_request_openai_gateway(
headers={"X-Gateway-Internal-Key": "test-internal-secret"}
),
cfg,
)
is True
)
assert (
svc.internal_bypass_allowed(
_make_request_openai_gateway(
headers={"X-Gateway-Internal-Key": "wrong-secret"}
),
cfg,
)
is False
)
assert (
svc.internal_bypass_allowed(
_make_request_openai_gateway(
headers={"Authorization": "Bearer test-internal-secret"}
),
cfg,
)
is False
)
assert svc.internal_bypass_allowed(_make_request_openai_gateway(), cfg) is False
finally:
set_setting("gateway_internal_key", "")
class FakeResponsesResp_openai_gateway(FakeResp_openai_gateway):
def __init__(self, status=200, payload=None, events=None, **kw):
super().__init__(status=status, payload=payload, **kw)
self._events = events or []
async def aiter_lines(self):
for event in self._events:
yield f"data: {json.dumps(event)}"
yield "data: [DONE]"
def _resp_obj_openai_gateway(model):
return {
"id": "resp_1",
"object": "response",
"created_at": 7,
"status": "completed",
"model": model,
"output": [
{
"type": "message",
"role": "assistant",
"content": [{"type": "output_text", "text": "hi there"}],
}
],
"usage": {"input_tokens": 5, "output_tokens": 7, "total_tokens": 12},
}
def _resp_events_openai_gateway(model):
return [
{"type": "response.created", "response": {"id": "resp_1"}},
{"type": "response.output_text.delta", "delta": "hi "},
{"type": "response.output_text.delta", "delta": "there"},
{"type": "response.completed", "response": _resp_obj_openai_gateway(model)},
]
def _resp_tool_obj_openai_gateway(model):
obj = _resp_obj_openai_gateway(model)
obj["output"].append(
{
"type": "function_call",
"call_id": "call_7",
"name": "ext_bash",
"arguments": "{}",
}
)
return obj
def _resp_tool_events_openai_gateway(model):
return [
{"type": "response.created", "response": {"id": "resp_1"}},
{
"type": "response.output_item.added",
"output_index": 1,
"item": {
"type": "function_call",
"call_id": "call_7",
"name": "ext_bash",
"arguments": "",
},
},
{
"type": "response.function_call_arguments.delta",
"item_id": "call_7",
"output_index": 1,
"delta": "{}",
},
{
"type": "response.completed",
"response": _resp_tool_obj_openai_gateway(model),
},
]
class FakeResponsesClient_openai_gateway:
behavior = "ok"
def __init__(self, *a, **k):
self.calls = []
self.last_headers = {}
def build_request(self, method, url, headers=None, json=None, content=None):
return FakeRequest(method, url, json, headers)
async def send(self, request, stream=False):
self.calls.append((request.url, request.json_body))
self.last_headers = dict(request.headers or {})
body = request.json_body or {}
if self.behavior == "error-envelope":
return FakeResp_openai_gateway(
status=429,
payload={
"error": {
"message": "slow down",
"type": "rate_limit",
"code": "rate_limit_exceeded",
}
},
)
if self.behavior == "failed-object":
return FakeResponsesResp_openai_gateway(
payload={
"object": "response",
"status": "failed",
"error": {"message": "boom"},
}
)
if self.behavior == "attestation-denied":
return FakeResp_openai_gateway(
status=403,
payload={
"error": {
"message": "This API key is billed to a Muse Code "
"subscription. Muse Code subscriptions cover requests "
"from the Muse Code client only.",
"type": "permission_error",
"code": "client_attestation_denied",
}
},
)
model = body.get("model", "")
if self.behavior == "tool-call":
return FakeResponsesResp_openai_gateway(
payload=_resp_tool_obj_openai_gateway(model),
events=_resp_tool_events_openai_gateway(model),
)
return FakeResponsesResp_openai_gateway(
payload=_resp_obj_openai_gateway(model),
events=_resp_events_openai_gateway(model),
)
async def post(self, url, headers=None, json=None, timeout=None):
return await self.send(FakeRequest("POST", url, json), stream=False)
async def aclose(self):
pass
def _setup_responses_route_openai_gateway():
from devplacepy.services.openai_gateway import routing
routing.provider_store.set(
routing.ProviderIn(
name="gwmeta",
base_url="https://api.meta.ai/v1/responses",
api_key="sk-test",
protocol="responses",
)
)
routing.model_store.set(
routing.ModelRouteIn(
source_model="meta-chat",
provider="gwmeta",
target_model="muse-spark-1.3-contributor",
kind="chat",
)
)
def _teardown_responses_route_openai_gateway():
from devplacepy.services.openai_gateway import routing
routing.model_store.remove("meta-chat")
routing.provider_store.remove("gwmeta")
def _chat_body_openai_gateway(**over):
body = {
"model": "meta-chat",
"messages": [
{"role": "system", "content": "Be terse."},
{"role": "user", "content": "hi"},
],
}
body.update(over)
return body
def test_responses_protocol_transpiles_request_and_response(local_db, monkeypatch):
monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeResponsesClient_openai_gateway)
FakeResponsesClient_openai_gateway.behavior = "ok"
_setup_responses_route_openai_gateway()
try:
svc = GatewayService()
cfg = svc.effective_config()
cfg["gateway_vision_enabled"] = False
rt = svc.runtime()
resp = run_async(
rt.handle_chat(
_chat_body_openai_gateway(), cfg, ("guest", "test"), "test", "default"
)
)
url, sent = rt._client.calls[-1]
assert str(url) == "https://api.meta.ai/v1/responses"
assert sent["model"] == "muse-spark-1.3-contributor"
assert "messages" not in sent
assert "Be terse." in sent["instructions"]
assert sent["input"] == [
{"type": "message", "role": "user", "content": "hi"}
]
assert sent["reasoning"] == {"effort": "minimal"}
assert resp.status_code == 200
payload = json.loads(bytes(resp.body).decode())
assert payload["object"] == "chat.completion"
assert payload["choices"][0]["message"]["content"] == "hi there"
assert payload["choices"][0]["finish_reason"] == "stop"
assert payload["usage"]["prompt_tokens"] == 5
assert resp.headers["X-Gateway-Model"] == "muse-spark-1.3-contributor"
finally:
_teardown_responses_route_openai_gateway()
def test_muse_profile_headers_reach_the_wire(local_db, monkeypatch):
from devplacepy.services.openai_gateway import routing
monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeResponsesClient_openai_gateway)
FakeResponsesClient_openai_gateway.behavior = "ok"
routing.provider_store.set(
routing.ProviderIn(
name="wiremeta",
base_url="https://api.meta.ai/v1/responses",
api_key="sk-test",
protocol="responses",
client_profile="muse",
)
)
routing.model_store.set(
routing.ModelRouteIn(
source_model="wire-muse",
provider="wiremeta",
target_model="muse-spark",
kind="chat",
)
)
try:
svc = GatewayService()
cfg = svc.effective_config()
cfg["gateway_vision_enabled"] = False
rt = svc.runtime()
logged = []
resp = run_async(
rt.handle_chat(
{"model": "wire-muse", "messages": [{"role": "user", "content": "hi"}]},
cfg,
("guest", "test"),
"test",
"default",
log=logged.append,
)
)
assert resp.status_code == 200
sent_headers = rt._client.last_headers
assert sent_headers["X-Client-Id"] == "tbh:exec"
assert sent_headers["User-Agent"].startswith("muse-build/")
session = sent_headers["X-Tbh-Session-Id"]
assert session
assert sent_headers["X-Meta-Ai-Gateway-Session-Id"] == session
assert sent_headers["X-Tbh-Thread-Id"] == session
assert sent_headers["X-Tbh-Root-Session-Id"] == session
assert any("(client profile 'muse')" in line for line in logged)
finally:
routing.model_store.remove("wire-muse")
routing.provider_store.remove("wiremeta")
def _bash_tool_body_openai_gateway(**over):
body = _chat_body_openai_gateway(**over)
body["tools"] = [
{
"type": "function",
"function": {
"name": "bash",
"description": "run shell",
"parameters": {"type": "object", "properties": {}},
},
}
]
return body
def test_meta_tool_names_aliased_on_wire_and_restored(local_db, monkeypatch):
monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeResponsesClient_openai_gateway)
FakeResponsesClient_openai_gateway.behavior = "tool-call"
_setup_responses_route_openai_gateway()
try:
svc = GatewayService()
cfg = svc.effective_config()
cfg["gateway_vision_enabled"] = False
rt = svc.runtime()
logged = []
resp = run_async(
rt.handle_chat(
_bash_tool_body_openai_gateway(),
cfg,
("guest", "test"),
"test",
"default",
log=logged.append,
)
)
_, sent = rt._client.calls[-1]
assert sent["tools"][0]["name"] == "ext_bash"
assert any("bash->ext_bash" in line for line in logged)
assert resp.status_code == 200
payload = json.loads(bytes(resp.body).decode())
calls = payload["choices"][0]["message"]["tool_calls"]
assert calls[0]["function"]["name"] == "bash"
assert payload["choices"][0]["finish_reason"] == "tool_calls"
finally:
FakeResponsesClient_openai_gateway.behavior = "ok"
_teardown_responses_route_openai_gateway()
def test_meta_tool_names_restored_on_stream_chunks(local_db, monkeypatch):
monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeResponsesClient_openai_gateway)
FakeResponsesClient_openai_gateway.behavior = "tool-call"
_setup_responses_route_openai_gateway()
try:
svc = GatewayService()
cfg = svc.effective_config()
cfg["gateway_vision_enabled"] = False
rt = svc.runtime()
resp = run_async(
rt.handle_chat(
_bash_tool_body_openai_gateway(stream=True),
cfg,
("guest", "test"),
"test",
"default",
)
)
async def drain():
out = []
async for chunk in resp.body_iterator:
out.append(chunk if isinstance(chunk, str) else chunk.decode())
return "".join(out)
body = run_async(drain())
assert '"name": "bash"' in body
assert "ext_bash" not in body
assert "[DONE]" in body
finally:
FakeResponsesClient_openai_gateway.behavior = "ok"
_teardown_responses_route_openai_gateway()
def _setup_chat_meta_route_openai_gateway():
from devplacepy.services.openai_gateway import routing
routing.provider_store.set(
routing.ProviderIn(
name="gwmetacc",
base_url="https://api.meta.ai/v1/chat/completions",
api_key="sk-test",
)
)
routing.model_store.set(
routing.ModelRouteIn(
source_model="meta-chat-cc",
provider="gwmetacc",
target_model="muse-spark",
kind="chat",
)
)
def _teardown_chat_meta_route_openai_gateway():
from devplacepy.services.openai_gateway import routing
routing.model_store.remove("meta-chat-cc")
routing.provider_store.remove("gwmetacc")
def _tai_tool_body_openai_gateway(**over):
# Mirrors a real tai/detz turn: core tools are always sent, so "shell"
# (a Pi-verified gate trigger missing from the denylist until
# 2026-11-10) rides every request alongside the lazy "read_file".
body = _chat_body_openai_gateway(**over)
body["model"] = "meta-chat-cc"
body["tools"] = [
{
"type": "function",
"function": {
"name": "shell",
"description": "run a shell command",
"parameters": {"type": "object", "properties": {}},
},
},
{
"type": "function",
"function": {
"name": "read_file",
"description": "read a file",
"parameters": {"type": "object", "properties": {}},
},
},
{
"type": "function",
"function": {
"name": "web_fetch",
"description": "fetch a URL",
"parameters": {"type": "object", "properties": {}},
},
},
]
return body
class FakeChatMetaStreamResp_openai_gateway(FakeResp_openai_gateway):
def __init__(self, payload):
super().__init__(status=200, payload=payload)
self._payload = payload
async def aiter_lines(self):
calls = self._payload["choices"][0]["message"]["tool_calls"]
yield (
'data: {"id":"chatcmpl-x","object":"chat.completion.chunk",'
'"model":"m","choices":[{"index":0,"delta":{"role":"assistant"}}]}'
)
for pos, call in enumerate(calls):
name = call["function"]["name"]
yield (
'data: {"id":"chatcmpl-x","object":"chat.completion.chunk",'
'"model":"m","choices":[{"index":0,"delta":{"tool_calls":'
'[{"index":%d,"id":"call_%d","function":{"name":"%s",'
'"arguments":"{}"}}]}}]}' % (pos, pos, name)
)
yield (
'data: {"id":"chatcmpl-x","object":"chat.completion.chunk",'
'"model":"m","choices":[{"index":0,"delta":{},'
'"finish_reason":"tool_calls"}]}'
)
yield "data: [DONE]"
class FakeChatMetaClient_openai_gateway:
def __init__(self, *a, **k):
self.calls = []
def build_request(self, method, url, headers=None, json=None, content=None):
return FakeRequest(method, url, json, headers)
async def send(self, request, stream=False):
self.calls.append((request.url, request.json_body))
body = request.json_body or {}
wire_names = [
(tool.get("function") or {}).get("name")
for tool in body.get("tools") or []
]
payload = {
"id": "chatcmpl-x",
"object": "chat.completion",
"model": body.get("model"),
"choices": [
{
"index": 0,
"message": {
"role": "assistant",
"content": "",
"tool_calls": [
{
"id": "call_7",
"type": "function",
"function": {"name": name, "arguments": "{}"},
}
for name in wire_names
],
},
"finish_reason": "tool_calls",
}
],
"usage": {"prompt_tokens": 5, "completion_tokens": 2, "total_tokens": 7},
}
if stream:
return FakeChatMetaStreamResp_openai_gateway(payload)
return FakeResp_openai_gateway(
payload=payload, content=json.dumps(payload).encode()
)
async def post(self, url, headers=None, json=None, timeout=None):
return await self.send(FakeRequest("POST", url, json), stream=False)
async def aclose(self):
pass
def test_meta_tool_names_aliased_on_chat_completions_wire(local_db, monkeypatch):
monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeChatMetaClient_openai_gateway)
_setup_chat_meta_route_openai_gateway()
try:
svc = GatewayService()
cfg = svc.effective_config()
cfg["gateway_vision_enabled"] = False
rt = svc.runtime()
logged = []
resp = run_async(
rt.handle_chat(
_tai_tool_body_openai_gateway(),
cfg,
("guest", "test"),
"test",
"default",
log=logged.append,
)
)
_, sent = rt._client.calls[-1]
wire = [tool["function"]["name"] for tool in sent["tools"]]
assert wire == ["ext_shell", "ext_read_file", "ext_web_fetch"]
assert any("shell->ext_shell" in line for line in logged)
assert any("read_file->ext_read_file" in line for line in logged)
assert any("web_fetch->ext_web_fetch" in line for line in logged)
assert resp.status_code == 200
payload = json.loads(bytes(resp.body).decode())
calls = payload["choices"][0]["message"]["tool_calls"]
assert [call["function"]["name"] for call in calls] == [
"shell",
"read_file",
"web_fetch",
]
assert payload["choices"][0]["finish_reason"] == "tool_calls"
finally:
_teardown_chat_meta_route_openai_gateway()
def test_meta_tool_names_restored_on_chat_completions_stream(
local_db, monkeypatch
):
monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeChatMetaClient_openai_gateway)
_setup_chat_meta_route_openai_gateway()
try:
svc = GatewayService()
cfg = svc.effective_config()
cfg["gateway_vision_enabled"] = False
rt = svc.runtime()
resp = run_async(
rt.handle_chat(
_tai_tool_body_openai_gateway(stream=True),
cfg,
("guest", "test"),
"test",
"default",
)
)
async def drain():
out = []
async for chunk in resp.body_iterator:
out.append(chunk if isinstance(chunk, str) else chunk.decode())
return "".join(out)
body = run_async(drain())
_, sent = rt._client.calls[-1]
wire = [tool["function"]["name"] for tool in sent["tools"]]
assert wire == ["ext_shell", "ext_read_file", "ext_web_fetch"]
assert '"name": "shell"' in body
assert '"name": "read_file"' in body
assert '"name": "web_fetch"' in body
assert "ext_shell" not in body
assert "ext_read_file" not in body
assert "ext_web_fetch" not in body
assert "[DONE]" in body
finally:
_teardown_chat_meta_route_openai_gateway()
def test_responses_protocol_stream_relays_chat_chunks(local_db, monkeypatch):
monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeResponsesClient_openai_gateway)
FakeResponsesClient_openai_gateway.behavior = "ok"
_setup_responses_route_openai_gateway()
try:
svc = GatewayService()
cfg = svc.effective_config()
cfg["gateway_vision_enabled"] = False
rt = svc.runtime()
resp = run_async(
rt.handle_chat(
_chat_body_openai_gateway(stream=True),
cfg,
("guest", "test"),
"test",
"default",
)
)
async def drain():
out = []
async for chunk in resp.body_iterator:
out.append(chunk if isinstance(chunk, str) else chunk.decode())
return "".join(out)
body = run_async(drain())
assert "chat.completion.chunk" in body
assert "hi " in body and "there" in body
assert '"usage"' not in body
assert "[DONE]" in body
finally:
_teardown_responses_route_openai_gateway()
def test_responses_protocol_stream_relays_usage_when_asked(local_db, monkeypatch):
monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeResponsesClient_openai_gateway)
FakeResponsesClient_openai_gateway.behavior = "ok"
_setup_responses_route_openai_gateway()
try:
svc = GatewayService()
cfg = svc.effective_config()
cfg["gateway_vision_enabled"] = False
rt = svc.runtime()
resp = run_async(
rt.handle_chat(
_chat_body_openai_gateway(
stream=True, stream_options={"include_usage": True}
),
cfg,
("guest", "test"),
"test",
"default",
)
)
_, sent = rt._client.calls[-1]
assert "stream_options" not in sent
async def drain():
out = []
async for chunk in resp.body_iterator:
out.append(chunk if isinstance(chunk, str) else chunk.decode())
return "".join(out)
body = run_async(drain())
assert '"usage"' in body
assert '"prompt_tokens": 5' in body
usage_chunks = [
json.loads(line[len("data:"):])
for line in body.splitlines()
if line.startswith("data: {") and '"usage"' in line
]
assert len(usage_chunks) == 1
assert usage_chunks[0]["choices"] == []
finally:
_teardown_responses_route_openai_gateway()
def test_responses_protocol_relays_error_envelope(local_db, monkeypatch):
monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeResponsesClient_openai_gateway)
FakeResponsesClient_openai_gateway.behavior = "error-envelope"
_setup_responses_route_openai_gateway()
try:
svc = GatewayService()
cfg = svc.effective_config()
cfg["gateway_vision_enabled"] = False
rt = svc.runtime()
resp = run_async(
rt.handle_chat(
_chat_body_openai_gateway(), cfg, ("guest", "test"), "test", "default"
)
)
assert resp.status_code == 429
payload = json.loads(bytes(resp.body).decode())
assert payload["error"]["message"] == "slow down"
assert payload["error"]["code"] == "rate_limit_exceeded"
finally:
FakeResponsesClient_openai_gateway.behavior = "ok"
_teardown_responses_route_openai_gateway()
def test_responses_protocol_attestation_denied_logs_guidance(
local_db, monkeypatch
):
monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeResponsesClient_openai_gateway)
FakeResponsesClient_openai_gateway.behavior = "attestation-denied"
_setup_responses_route_openai_gateway()
try:
svc = GatewayService()
cfg = svc.effective_config()
cfg["gateway_vision_enabled"] = False
rt = svc.runtime()
logged = []
resp = run_async(
rt.handle_chat(
_chat_body_openai_gateway(),
cfg,
("guest", "test"),
"test",
"default",
log=logged.append,
)
)
assert resp.status_code == 403
payload = json.loads(bytes(resp.body).decode())
assert payload["error"]["code"] == "client_attestation_denied"
assert any("pay-as-you-go" in line for line in logged)
finally:
FakeResponsesClient_openai_gateway.behavior = "ok"
_teardown_responses_route_openai_gateway()
def test_upstream_auth_guidance_covers_401_and_403():
denied = FakeResp_openai_gateway(
status=403,
payload={
"error": {
"message": "subscription only",
"type": "permission_error",
"code": "client_attestation_denied",
}
},
)
guidance = gwmod._upstream_auth_guidance(denied, denied.json())
assert "pay-as-you-go" in guidance
assert "aliasing" in guidance
assert "MUSE_RESERVED_TOOL_NAMES" in guidance
other_denied = FakeResp_openai_gateway(status=403, payload={"error": "nope"})
assert (
gwmod._upstream_auth_guidance(other_denied, other_denied.json()) is None
)
unauthed = FakeResp_openai_gateway(
status=401, payload={"error": {"message": "User not found.", "code": 401}}
)
guidance = gwmod._upstream_auth_guidance(unauthed, unauthed.json())
assert "api_key" in guidance
assert "User not found." in guidance
server_error = FakeResp_openai_gateway(status=500, payload={"error": "x"})
assert (
gwmod._upstream_auth_guidance(server_error, server_error.json()) is None
)
def test_should_fallback_model_unavailable_but_not_bad_request():
unavailable = FakeResp_openai_gateway(
status=400,
payload={"error": {"message": "Model is unavailable."}},
)
assert (
gwmod._should_fallback(unavailable, None, {"circuit_open": False}) is True
)
malformed = FakeResp_openai_gateway(
status=400, payload={"error": "bad schema"}
)
assert (
gwmod._should_fallback(malformed, None, {"circuit_open": False}) is False
)
def test_responses_protocol_failed_object_is_502(local_db, monkeypatch):
monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeResponsesClient_openai_gateway)
FakeResponsesClient_openai_gateway.behavior = "failed-object"
_setup_responses_route_openai_gateway()
try:
svc = GatewayService()
cfg = svc.effective_config()
cfg["gateway_vision_enabled"] = False
rt = svc.runtime()
resp = run_async(
rt.handle_chat(
_chat_body_openai_gateway(), cfg, ("guest", "test"), "test", "default"
)
)
assert resp.status_code == 502
finally:
FakeResponsesClient_openai_gateway.behavior = "ok"
_teardown_responses_route_openai_gateway()
def test_responses_protocol_client_reasoning_effort_passes_through(
local_db, monkeypatch
):
monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeResponsesClient_openai_gateway)
FakeResponsesClient_openai_gateway.behavior = "ok"
_setup_responses_route_openai_gateway()
try:
svc = GatewayService()
cfg = svc.effective_config()
cfg["gateway_vision_enabled"] = False
rt = svc.runtime()
run_async(
rt.handle_chat(
_chat_body_openai_gateway(reasoning_effort="high"),
cfg,
("guest", "test"),
"test",
"default",
)
)
_, sent = rt._client.calls[-1]
assert sent["reasoning"] == {"effort": "high"}
finally:
_teardown_responses_route_openai_gateway()
def test_responses_protocol_default_provider(local_db, monkeypatch):
monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeResponsesClient_openai_gateway)
FakeResponsesClient_openai_gateway.behavior = "ok"
svc = GatewayService()
cfg = svc.effective_config()
cfg["gateway_vision_enabled"] = False
cfg["gateway_upstream_url"] = "https://api.meta.ai/v1/responses"
cfg["gateway_upstream_protocol"] = "responses"
cfg["gateway_model"] = "muse-spark-1.3-contributor"
rt = svc.runtime()
resp = run_async(
rt.handle_chat(
{"messages": [{"role": "user", "content": "hi"}]},
cfg,
("guest", "test"),
"test",
"default",
)
)
url, sent = rt._client.calls[-1]
assert str(url) == "https://api.meta.ai/v1/responses"
assert sent["input"] == [{"type": "message", "role": "user", "content": "hi"}]
assert resp.status_code == 200
payload = json.loads(bytes(resp.body).decode())
assert payload["object"] == "chat.completion"
EMULATED_CALL_TEXT_openai_gateway = (
"Sure.\n<tool_call>\n"
'{"name": "bash", "arguments": {"command": "ls"}}\n'
"</tool_call>"
)
BASH_TOOL_openai_gateway = {
"type": "function",
"function": {
"name": "bash",
"description": "Run a shell command",
"parameters": {
"type": "object",
"properties": {"command": {"type": "string"}},
"required": ["command"],
},
},
}
def _emulated_obj_openai_gateway(model, text):
obj = _resp_obj_openai_gateway(model)
obj["output"][0]["content"][0]["text"] = text
return obj
class FakeEmulatedClient_openai_gateway(FakeResponsesClient_openai_gateway):
text = EMULATED_CALL_TEXT_openai_gateway
async def send(self, request, stream=False):
self.calls.append((request.url, request.json_body))
model = (request.json_body or {}).get("model", "")
obj = _emulated_obj_openai_gateway(model, self.text)
events = [{"type": "response.created", "response": {"id": "resp_1"}}]
for i in range(0, len(self.text), 4):
events.append(
{"type": "response.output_text.delta", "delta": self.text[i : i + 4]}
)
events.append({"type": "response.completed", "response": obj})
return FakeResponsesResp_openai_gateway(payload=obj, events=events)
def _setup_emulated_route_openai_gateway():
from devplacepy.services.openai_gateway import routing
routing.provider_store.set(
routing.ProviderIn(
name="gwmeta",
base_url="https://api.meta.ai/v1/responses",
api_key="sk-test",
protocol="responses",
)
)
routing.model_store.set(
routing.ModelRouteIn(
source_model="meta-chat",
provider="gwmeta",
target_model="muse-spark-1.3-contributor",
kind="chat",
tools_mode="emulated",
)
)
def _emulated_chat_openai_gateway(monkeypatch, body, **tools):
monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeEmulatedClient_openai_gateway)
_setup_emulated_route_openai_gateway()
svc = GatewayService()
cfg = svc.effective_config()
cfg["gateway_vision_enabled"] = False
rt = svc.runtime()
resp = run_async(rt.handle_chat(body, cfg, ("guest", "test"), "test", "default"))
return rt, resp
def test_emulated_tools_meta_responses_nonstream(local_db, monkeypatch):
body = _chat_body_openai_gateway(tools=[BASH_TOOL_openai_gateway], tool_choice="auto")
try:
rt, resp = _emulated_chat_openai_gateway(monkeypatch, body)
url, sent = rt._client.calls[-1]
assert str(url) == "https://api.meta.ai/v1/responses"
assert "tools" not in sent and "tool_choice" not in sent
assert "<tools>" in sent["instructions"] and '"bash"' in sent["instructions"]
assert resp.status_code == 200
message = json.loads(bytes(resp.body).decode())["choices"][0]
assert message["finish_reason"] == "tool_calls"
call = message["message"]["tool_calls"][0]
assert call["function"]["name"] == "bash"
assert json.loads(call["function"]["arguments"]) == {"command": "ls"}
assert message["message"]["content"] == "Sure."
finally:
_teardown_responses_route_openai_gateway()
def test_emulated_tools_meta_responses_stream(local_db, monkeypatch):
body = _chat_body_openai_gateway(
stream=True, tools=[BASH_TOOL_openai_gateway]
)
try:
rt, resp = _emulated_chat_openai_gateway(monkeypatch, body)
async def drain():
out = []
async for chunk in resp.body_iterator:
out.append(chunk if isinstance(chunk, str) else chunk.decode())
return "".join(out)
raw = run_async(drain())
chunks = [
json.loads(line[5:])
for line in raw.splitlines()
if line.startswith("data:") and "[DONE]" not in line
]
text = "".join(
c["choices"][0]["delta"].get("content", "")
for c in chunks
if c.get("choices")
)
calls = [
tc
for c in chunks
for choice in c.get("choices") or []
for tc in choice["delta"].get("tool_calls") or []
]
assert "<tool_call>" not in text and text.strip() == "Sure."
assert len(calls) == 1 and calls[0]["function"]["name"] == "bash"
finishes = [
c["choices"][0]["finish_reason"]
for c in chunks
if c.get("choices") and c["choices"][0].get("finish_reason")
]
assert finishes[-1] == "tool_calls"
assert raw.rstrip().endswith("[DONE]")
finally:
_teardown_responses_route_openai_gateway()
def test_emulated_tools_convert_history_for_the_meta_transpiler(local_db, monkeypatch):
history = [
{"role": "user", "content": "go"},
{
"role": "assistant",
"content": None,
"tool_calls": [
{
"id": "call_1",
"type": "function",
"function": {"name": "bash", "arguments": '{"command": "ls"}'},
}
],
},
{"role": "tool", "tool_call_id": "call_1", "content": "a.txt"},
]
body = _chat_body_openai_gateway(messages=history, tools=[BASH_TOOL_openai_gateway])
try:
rt, resp = _emulated_chat_openai_gateway(monkeypatch, body)
_, sent = rt._client.calls[-1]
assert resp.status_code == 200
assert all(item.get("type") == "message" for item in sent["input"])
joined = json.dumps(sent["input"])
assert "<tool_call>" in joined and "<tool_response>" in joined and "a.txt" in joined
assert "function_call" not in joined
finally:
_teardown_responses_route_openai_gateway()
def test_emulated_tools_serve_large_catalogs_with_no_cap(local_db, monkeypatch):
tools = [
{
"type": "function",
"function": {
"name": f"tool_{index}",
"description": f"tool number {index}",
"parameters": {"type": "object", "properties": {}, "required": []},
},
}
for index in range(200)
]
body = _chat_body_openai_gateway(tools=tools, tool_choice="auto")
try:
rt, resp = _emulated_chat_openai_gateway(monkeypatch, body)
assert resp.status_code == 200
assert len(rt._client.calls) == 1
_, sent = rt._client.calls[-1]
assert "tools" not in sent and "tool_choice" not in sent
assert '"name": "tool_0"' in sent["instructions"]
assert '"name": "tool_199"' in sent["instructions"]
assert len(rt._catalog_cache._entries) == 1
finally:
_teardown_responses_route_openai_gateway()
def test_native_route_keeps_forwarding_tools(local_db, monkeypatch):
monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeResponsesClient_openai_gateway)
FakeResponsesClient_openai_gateway.behavior = "ok"
_setup_responses_route_openai_gateway()
try:
svc = GatewayService()
cfg = svc.effective_config()
cfg["gateway_vision_enabled"] = False
rt = svc.runtime()
run_async(
rt.handle_chat(
_chat_body_openai_gateway(tools=[BASH_TOOL_openai_gateway]),
cfg,
("guest", "test"),
"test",
"default",
)
)
_, sent = rt._client.calls[-1]
assert "tools" in sent
assert "<tools>" not in sent["instructions"]
finally:
_teardown_responses_route_openai_gateway()