# retoor import json from starlette.requests import Request from tests.conftest import BASE_URL, run_async from devplacepy.database import get_table, set_setting from devplacepy.utils import generate_uid import devplacepy.services.openai_gateway.gateway as gwmod from devplacepy.services.openai_gateway import GatewayService from devplacepy.services.openai_gateway import auth_throttle class FakeResp_openai_gateway: def __init__( self, status=200, payload=None, ctype="application/json", content=b"", extra_headers=None, ): self.status_code = status self._payload = payload self.text = json.dumps(payload) if payload is not None else "" self.headers = {"content-type": ctype, **(extra_headers or {})} self.content = content def json(self): if self._payload is None: raise ValueError("no json") return self._payload async def aread(self): return self.content or self.text.encode() async def aclose(self): pass async def aiter_lines(self): payload = self._payload or {} chunk_id = payload.get("id", "x") model = payload.get("model", "") try: content = payload["choices"][0]["message"].get("content") or "" except (KeyError, IndexError, TypeError): content = "" try: reasoning = payload["choices"][0]["message"].get("reasoning") or "" except (KeyError, IndexError, TypeError): reasoning = "" def frame(delta=None, finish=None, usage=None): body = {"id": chunk_id, "object": "chat.completion.chunk", "model": model} if usage is not None: body["choices"] = [] body["usage"] = usage else: body["choices"] = [{"index": 0, "delta": delta or {}, "finish_reason": finish}] return f"data: {json.dumps(body)}" yield frame({"role": "assistant"}) for i in range(0, len(reasoning), 5): yield frame({"reasoning": reasoning[i : i + 5]}) for i in range(0, len(content), 5): yield frame({"content": content[i : i + 5]}) yield frame({}, finish="stop") if payload.get("usage"): yield frame(usage=payload["usage"]) yield "data: [DONE]" class FakeRequest: def __init__(self, method, url, json_body, headers=None): self.method = method self.url = url self.json_body = json_body self.headers = headers or {} self.extensions = {} class FakeClient_openai_gateway: def __init__(self, *a, **k): self.calls = [] def build_request(self, method, url, headers=None, json=None, content=None): return FakeRequest(method, url, json) async def send(self, request, stream=False): self.calls.append((request.url, request.json_body)) body = request.json_body or {} return FakeResp_openai_gateway( payload={ "id": "x", "model": body.get("model"), "choices": [{"message": {"content": "hi there"}}], } ) async def post(self, url, headers=None, json=None, timeout=None): self.calls.append((url, json)) return FakeResp_openai_gateway( payload={ "id": "x", "model": json.get("model"), "choices": [{"message": {"content": "hi there"}}], } ) async def request(self, method, url, headers=None, content=None): return FakeResp_openai_gateway(payload={"ok": True}) async def aclose(self): pass def _make_request_openai_gateway(headers=None, cookies=None): headers = headers or {} raw = [(k.lower().encode(), v.encode()) for k, v in headers.items()] if cookies: raw.append( (b"cookie", "; ".join(f"{k}={v}" for k, v in cookies.items()).encode()) ) return Request( { "type": "http", "method": "POST", "path": "/openai/v1/chat/completions", "query_string": b"", "headers": raw, "state": {}, } ) def _make_admin_openai_gateway(role="Admin"): username = f"gw_{generate_uid()[:8]}" api_key = generate_uid() get_table("users").insert( { "uid": generate_uid(), "username": username, "terms_version": "1", "email": f"{username}@t.dev", "api_key": api_key, "role": role, "is_active": True, } ) return username, api_key def _config(page, **fields): page.request.post(f"{BASE_URL}/admin/services/openai/config", form=fields) def test_model_is_forced(local_db, monkeypatch): monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeClient_openai_gateway) svc = GatewayService() cfg = svc.effective_config() cfg["gateway_force_model"] = True cfg["gateway_model"] = "deepseek-chat" rt = svc.runtime() run_async( rt.handle_chat( {"model": "gpt-4", "messages": [{"role": "user", "content": "hi"}]}, cfg, ("guest", "test"), "test", "default", ) ) assert rt._client.calls[-1][1]["model"] == "deepseek-chat" def test_model_route_overrides_upstream(local_db, monkeypatch): from devplacepy.services.openai_gateway import routing monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeClient_openai_gateway) routing.provider_store.set( routing.ProviderIn( name="gwroute", base_url="https://routed.example/v1/chat/completions", api_key="routed-key", ) ) routing.model_store.set( routing.ModelRouteIn( source_model="custom-pro", provider="gwroute", target_model="vendor/pro", kind="chat", price_output_per_m=9.0, ) ) try: svc = GatewayService() cfg = svc.effective_config() cfg["gateway_vision_enabled"] = False rt = svc.runtime() run_async( rt.handle_chat( {"model": "custom-pro", "messages": [{"role": "user", "content": "hi"}]}, cfg, ("guest", "test"), "test", "default", ) ) url, body = rt._client.calls[-1] assert str(url) == "https://routed.example/v1/chat/completions" assert body["model"] == "vendor/pro" finally: routing.model_store.remove("custom-pro") routing.provider_store.remove("gwroute") def test_vision_rewrites_image_to_text(local_db, monkeypatch): monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeClient_openai_gateway) svc = GatewayService() cfg = svc.effective_config() cfg["gateway_vision_enabled"] = True cfg["gateway_vision_key"] = "" # no key -> placeholder text, still rewrites to str rt = svc.runtime() msgs = [ { "role": "user", "content": [ {"type": "text", "text": "what is this"}, {"type": "image_url", "image_url": {"url": "http://x/y.png"}}, ], } ] run_async(rt.handle_chat({"messages": msgs}, cfg, ("guest", "test"), "test", "default")) sent = rt._client.calls[-1][1]["messages"][0]["content"] assert isinstance(sent, str) and "vision" in sent.lower() def test_streaming_emits_sse(local_db, monkeypatch): monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeClient_openai_gateway) svc = GatewayService() cfg = svc.effective_config() rt = svc.runtime() resp = run_async( rt.handle_chat( {"messages": [{"role": "user", "content": "hi"}], "stream": True}, cfg, ("guest", "test"), "test", "default", ) ) async def drain(): out = [] async for chunk in resp.body_iterator: out.append(chunk if isinstance(chunk, str) else chunk.decode()) return "".join(out) body = run_async(drain()) assert "chat.completion.chunk" in body assert "[DONE]" in body def test_authorize_static_access_key(local_db): set_setting("gateway_require_auth", "1") set_setting("gateway_access_key", "topsecret") try: svc = GatewayService() assert svc.authorize(_make_request_openai_gateway(headers={"X-API-KEY": "topsecret"})) is True assert svc.authorize(_make_request_openai_gateway(headers={"X-API-KEY": "wrong"})) is False finally: set_setting("gateway_access_key", "") set_setting("gateway_require_auth", "1") auth_throttle.clear() def test_authorize_admin_and_user_toggles(local_db): set_setting("gateway_require_auth", "1") set_setting("gateway_access_key", "") set_setting("gateway_allow_admins", "1") set_setting("gateway_allow_users", "0") try: _, admin_key = _make_admin_openai_gateway(role="Admin") _, member_key = _make_admin_openai_gateway(role="Member") svc = GatewayService() assert ( svc.authorize(_make_request_openai_gateway(headers={"Authorization": f"Bearer {admin_key}"})) is True ) assert svc.authorize(_make_request_openai_gateway(headers={"X-API-KEY": member_key})) is False set_setting("gateway_allow_users", "1") assert svc.authorize(_make_request_openai_gateway(headers={"X-API-KEY": member_key})) is True finally: set_setting("gateway_allow_admins", "1") set_setting("gateway_allow_users", "1") set_setting("gateway_require_auth", "1") set_setting("gateway_access_key", "") auth_throttle.clear() def test_authorize_require_auth_off_is_open(local_db): set_setting("gateway_require_auth", "0") try: svc = GatewayService() assert svc.authorize(_make_request_openai_gateway()) is True finally: set_setting("gateway_require_auth", "1") def test_authorize_throttles_repeated_failed_auth_from_one_ip(local_db): import pytest from fastapi import HTTPException set_setting("gateway_require_auth", "1") set_setting("gateway_access_key", "") set_setting("gateway_auth_throttle_max_failures", "3") set_setting("gateway_auth_throttle_window_seconds", "60") auth_throttle.clear() try: svc = GatewayService() ip = "203.0.113.10" for _ in range(3): assert ( svc.authorize(_make_request_openai_gateway(headers={"X-Real-IP": ip})) is False ) with pytest.raises(HTTPException) as excinfo: svc.authorize(_make_request_openai_gateway(headers={"X-Real-IP": ip})) assert excinfo.value.status_code == 429 assert "Retry-After" in excinfo.value.headers finally: set_setting("gateway_auth_throttle_max_failures", "10") set_setting("gateway_auth_throttle_window_seconds", "60") set_setting("gateway_require_auth", "1") auth_throttle.clear() def test_authorize_throttle_does_not_affect_other_ips(local_db): set_setting("gateway_require_auth", "1") set_setting("gateway_access_key", "") set_setting("gateway_auth_throttle_max_failures", "3") set_setting("gateway_auth_throttle_window_seconds", "60") auth_throttle.clear() try: svc = GatewayService() throttled_ip = "203.0.113.20" other_ip = "203.0.113.21" for _ in range(3): svc.authorize(_make_request_openai_gateway(headers={"X-Real-IP": throttled_ip})) assert auth_throttle.is_throttled(throttled_ip, 3, 60) is True assert ( svc.authorize(_make_request_openai_gateway(headers={"X-Real-IP": other_ip})) is False ) assert auth_throttle.is_throttled(other_ip, 3, 60) is False finally: set_setting("gateway_auth_throttle_max_failures", "10") set_setting("gateway_auth_throttle_window_seconds", "60") set_setting("gateway_require_auth", "1") auth_throttle.clear() def test_authorize_valid_credentials_succeed_from_throttled_ip(local_db): set_setting("gateway_require_auth", "1") set_setting("gateway_access_key", "") set_setting("gateway_allow_users", "1") set_setting("gateway_auth_throttle_max_failures", "3") set_setting("gateway_auth_throttle_window_seconds", "60") auth_throttle.clear() try: svc = GatewayService() ip = "203.0.113.30" _, member_key = _make_admin_openai_gateway(role="Member") for _ in range(3): svc.authorize(_make_request_openai_gateway(headers={"X-Real-IP": ip})) assert auth_throttle.is_throttled(ip, 3, 60) is True assert ( svc.authorize( _make_request_openai_gateway( headers={"X-Real-IP": ip, "X-API-KEY": member_key} ) ) is True ) set_setting("gateway_access_key", "topsecret") assert ( svc.authorize( _make_request_openai_gateway( headers={"X-Real-IP": ip, "X-API-KEY": "topsecret"} ) ) is True ) finally: set_setting("gateway_access_key", "") set_setting("gateway_auth_throttle_max_failures", "10") set_setting("gateway_auth_throttle_window_seconds", "60") set_setting("gateway_require_auth", "1") auth_throttle.clear() def test_authorize_throttle_resets_after_window(local_db, monkeypatch): set_setting("gateway_require_auth", "1") set_setting("gateway_access_key", "") set_setting("gateway_auth_throttle_max_failures", "3") set_setting("gateway_auth_throttle_window_seconds", "60") auth_throttle.clear() try: now = [2000.0] monkeypatch.setattr(auth_throttle.time, "time", lambda: now[0]) svc = GatewayService() ip = "203.0.113.40" for _ in range(3): svc.authorize(_make_request_openai_gateway(headers={"X-Real-IP": ip})) assert auth_throttle.is_throttled(ip, 3, 60) is True now[0] += 61 assert ( svc.authorize(_make_request_openai_gateway(headers={"X-Real-IP": ip})) is False ) finally: set_setting("gateway_auth_throttle_max_failures", "10") set_setting("gateway_auth_throttle_window_seconds", "60") set_setting("gateway_require_auth", "1") auth_throttle.clear() class FakeEmbedClient_openai_gateway: def __init__(self, *a, **k): self.calls = [] def build_request(self, method, url, headers=None, json=None, content=None): return FakeRequest(method, url, json) async def send(self, request, stream=False): self.calls.append((request.url, request.json_body)) body = request.json_body or {} return FakeResp_openai_gateway( payload={ "object": "list", "model": body.get("model"), "data": [{"object": "embedding", "index": 0, "embedding": [0.1, 0.2]}], "usage": {"prompt_tokens": 5, "total_tokens": 5}, } ) async def aclose(self): pass def test_embeddings_remaps_alias_to_configured_model(local_db, monkeypatch): monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeEmbedClient_openai_gateway) monkeypatch.setenv("OPENROUTER_API_KEY", "test-embed-key") svc = GatewayService() cfg = svc.effective_config() cfg["gateway_force_model"] = False cfg["gateway_embed_enabled"] = True cfg["gateway_embed_model"] = "qwen/qwen3-embedding-8b" rt = svc.runtime() run_async( rt.handle_embeddings( {"model": "molodetz~embed", "input": "hello"}, cfg, ("guest", "test"), "test", "default", ) ) assert rt._client.calls[-1][1]["model"] == "qwen/qwen3-embedding-8b" def test_embeddings_success_records_ledger(local_db, monkeypatch): monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeEmbedClient_openai_gateway) monkeypatch.setenv("OPENROUTER_API_KEY", "test-embed-key") svc = GatewayService() cfg = svc.effective_config() cfg["gateway_force_model"] = True cfg["gateway_embed_enabled"] = True cfg["gateway_embed_model"] = "qwen/qwen3-embedding-8b" rt = svc.runtime() before = rt.embed_calls resp = run_async( rt.handle_embeddings( {"model": "molodetz~embed", "input": "hello"}, cfg, ("guest", "ledger_probe"), "test", "default", ) ) assert resp.status_code == 200 payload = json.loads(bytes(resp.body).decode()) assert payload["data"][0]["embedding"] == [0.1, 0.2] assert rt.embed_calls == before + 1 row = get_table("gateway_usage_ledger").find_one(owner_id="ledger_probe") assert row is not None assert row["backend"] == "embed" assert row["endpoint"] == "embeddings" assert row["requested_model"] == "molodetz~embed" assert row["model"] == "qwen/qwen3-embedding-8b" assert row["success"] == 1 def test_embeddings_disabled_returns_503(local_db, monkeypatch): monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeEmbedClient_openai_gateway) svc = GatewayService() cfg = svc.effective_config() cfg["gateway_embed_enabled"] = False rt = svc.runtime() resp = run_async( rt.handle_embeddings( {"input": "hello"}, cfg, ("guest", "test"), "test", "default" ) ) assert resp.status_code == 503 def test_embeddings_missing_key_returns_clean_error_not_unauthenticated_call( local_db, monkeypatch ): monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeEmbedClient_openai_gateway) monkeypatch.delenv("OPENROUTER_API_KEY", raising=False) monkeypatch.delenv("DEEPSEEK_API_KEY", raising=False) set_setting("gateway_api_key", "") set_setting("gateway_vision_key", "") set_setting("gateway_embed_key", "") try: svc = GatewayService() cfg = svc.effective_config() assert cfg["gateway_embed_key"] == "" cfg["gateway_embed_enabled"] = True rt = svc.runtime() resp = run_async( rt.handle_embeddings( {"input": "hello"}, cfg, ("guest", "no_key_probe"), "test", "default" ) ) assert resp.status_code == 503 payload = json.loads(bytes(resp.body).decode()) assert payload["error"]["type"] == "embeddings_not_configured" assert rt._client is None finally: set_setting("gateway_api_key", "") set_setting("gateway_vision_key", "") set_setting("gateway_embed_key", "") class FakeImageClient_openai_gateway: def __init__(self, *a, **k): self.calls = [] def build_request(self, method, url, headers=None, json=None, content=None): return FakeRequest(method, url, json) async def send(self, request, stream=False): self.calls.append((request.url, request.json_body)) body = request.json_body or {} return FakeResp_openai_gateway( payload={ "created": 1, "model": body.get("model"), "data": [{"b64_json": "aGVsbG8="}], "usage": {"cost": 0.05}, } ) async def aclose(self): pass def test_images_remaps_alias_to_configured_model(local_db, monkeypatch): monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeImageClient_openai_gateway) svc = GatewayService() cfg = svc.effective_config() cfg["gateway_force_model"] = False cfg["gateway_image_enabled"] = True cfg["gateway_image_model"] = "black-forest-labs/flux.2-pro" rt = svc.runtime() run_async( rt.handle_images( { "model": "molodetz-img-small", "prompt": "award emblem", "size": "512x512", }, cfg, ("guest", "test"), "test", "default", ) ) assert rt._client.calls[-1][1]["model"] == "black-forest-labs/flux.2-pro" def test_image_route_overrides_upstream(local_db, monkeypatch): from devplacepy.services.openai_gateway import routing monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeImageClient_openai_gateway) routing.provider_store.set( routing.ProviderIn( name="gwimg", base_url="https://routed.example/v1/chat/completions", api_key="img-key", ) ) routing.model_store.set( routing.ModelRouteIn( source_model="custom-img", provider="gwimg", target_model="vendor/flux-pro", kind="image", price_input_per_m=0.06, ) ) try: svc = GatewayService() cfg = svc.effective_config() rt = svc.runtime() run_async( rt.handle_images( { "model": "custom-img", "prompt": "trophy", "response_format": "b64_json", }, cfg, ("guest", "test"), "test", "default", ) ) url, body = rt._client.calls[-1] assert str(url) == "https://routed.example/v1/images" assert body["model"] == "vendor/flux-pro" finally: routing.model_store.remove("custom-img") routing.provider_store.remove("gwimg") def test_images_success_records_ledger(local_db, monkeypatch): monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeImageClient_openai_gateway) svc = GatewayService() cfg = svc.effective_config() cfg["gateway_image_enabled"] = True cfg["gateway_image_model"] = "black-forest-labs/flux.2-pro" rt = svc.runtime() before = rt.image_calls resp = run_async( rt.handle_images( {"model": "molodetz-img-small", "prompt": "badge"}, cfg, ("guest", "img_ledger"), "test", "default", ) ) assert resp.status_code == 200 payload = json.loads(bytes(resp.body).decode()) assert payload["data"][0]["b64_json"] == "aGVsbG8=" assert rt.image_calls == before + 1 row = get_table("gateway_usage_ledger").find_one(owner_id="img_ledger") assert row is not None assert row["backend"] == "image" assert row["endpoint"] == "images/generations" assert row["requested_model"] == "molodetz-img-small" assert row["model"] == "black-forest-labs/flux.2-pro" assert row["success"] == 1 assert float(row["cost_usd"]) == 0.05 def test_images_disabled_returns_503(local_db, monkeypatch): monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeImageClient_openai_gateway) svc = GatewayService() cfg = svc.effective_config() cfg["gateway_image_enabled"] = False rt = svc.runtime() resp = run_async( rt.handle_images( {"prompt": "badge"}, cfg, ("guest", "test"), "test", "default" ) ) assert resp.status_code == 503 def test_compute_cost_embed_branch(): from devplacepy.services.openai_gateway.usage import ( Pricing, compute_cost, normalize_usage, ) pricing = Pricing( chat_cache_hit_per_m=0.0, chat_cache_miss_per_m=0.0, chat_output_per_m=0.0, vision_input_per_m=0.0, vision_output_per_m=0.0, embed_input_per_m=0.01, ) usage = {"prompt_tokens": 1_000_000, "total_tokens": 1_000_000} norm = normalize_usage(usage) total, input_cost, output_cost, native = compute_cost( usage, norm, pricing, "embed" ) assert native is False assert output_cost == 0.0 assert abs(total - 0.01) < 1e-9 assert abs(input_cost - 0.01) < 1e-9 def test_app_reference_stored_in_ledger_for_chat(local_db, monkeypatch): monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeClient_openai_gateway) svc = GatewayService() cfg = svc.effective_config() cfg["gateway_force_model"] = True cfg["gateway_model"] = "deepseek-chat" rt = svc.runtime() run_async( rt.handle_chat( {"messages": [{"role": "user", "content": "ref_test"}]}, cfg, ("guest", "ref_probe"), "test-ua", "my-custom-app", ) ) row = get_table("gateway_usage_ledger").find_one(owner_id="ref_probe") assert row is not None assert row["app_reference"] == "my-custom-app" assert row["backend"] == "chat" def test_app_reference_defaults_when_not_passed(local_db, monkeypatch): monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeClient_openai_gateway) svc = GatewayService() cfg = svc.effective_config() cfg["gateway_force_model"] = True cfg["gateway_model"] = "deepseek-chat" rt = svc.runtime() run_async( rt.handle_chat( {"messages": [{"role": "user", "content": "default_test"}]}, cfg, ("guest", "default_probe"), "test-ua", "default", ) ) row = get_table("gateway_usage_ledger").find_one(owner_id="default_probe") assert row is not None assert row["app_reference"] == "default" def test_app_reference_stored_in_ledger_for_embeddings(local_db, monkeypatch): monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeEmbedClient_openai_gateway) monkeypatch.setenv("OPENROUTER_API_KEY", "test-embed-key") svc = GatewayService() cfg = svc.effective_config() cfg["gateway_embed_enabled"] = True cfg["gateway_embed_model"] = "qwen/qwen3-embedding-8b" rt = svc.runtime() run_async( rt.handle_embeddings( {"input": "hello"}, cfg, ("guest", "embed_ref"), "test-ua", "devplace-embed-test-v-1-0-0", ) ) row = get_table("gateway_usage_ledger").find_one(owner_id="embed_ref") assert row is not None assert row["app_reference"] == "devplace-embed-test-v-1-0-0" assert row["backend"] == "embed" def test_app_reference_stored_in_ledger_for_images(local_db, monkeypatch): monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeImageClient_openai_gateway) svc = GatewayService() cfg = svc.effective_config() cfg["gateway_image_enabled"] = True cfg["gateway_image_model"] = "black-forest-labs/flux.2-pro" rt = svc.runtime() run_async( rt.handle_images( {"prompt": "ref badge"}, cfg, ("guest", "img_ref"), "test-ua", "devplace-image-test-v-1-0-0", ) ) row = get_table("gateway_usage_ledger").find_one(owner_id="img_ref") assert row is not None assert row["app_reference"] == "devplace-image-test-v-1-0-0" assert row["backend"] == "image" def test_app_reference_column_exists(local_db): db = get_table("gateway_usage_ledger").db if "gateway_usage_ledger" not in db.tables: return rows = list(db.query("PRAGMA table_info('gateway_usage_ledger')")) column_names = [r["name"] for r in rows] assert "app_reference" in column_names def test_chat_unknown_model_falls_back_to_default(local_db, monkeypatch): monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeClient_openai_gateway) svc = GatewayService() cfg = svc.effective_config() cfg["gateway_force_model"] = False cfg["gateway_model"] = "deepseek-v4-flash" rt = svc.runtime() run_async( rt.handle_chat( {"model": "nonexistent-model-v99", "messages": [{"role": "user", "content": "hi"}]}, cfg, ("guest", "fallback_test_chat"), "test", "default", ) ) assert rt._client.calls[-1][1]["model"] == "deepseek-v4-flash" class FakeFallbackClient_openai_gateway: def __init__(self, *a, **k): self.calls = [] def build_request(self, method, url, headers=None, json=None, content=None): return FakeRequest(method, url, json) async def send(self, request, stream=False): self.calls.append((request.url, request.json_body)) body = request.json_body or {} model = body.get("model") if model == "primary-target": return FakeResp_openai_gateway(status=500, payload={"error": "boom"}) return FakeResp_openai_gateway( payload={ "id": "x", "model": model, "choices": [{"message": {"content": "fallback ok"}}], } ) async def aclose(self): pass class FakeAlwaysFailClient_openai_gateway: def __init__(self, *a, **k): self.calls = [] def build_request(self, method, url, headers=None, json=None, content=None): return FakeRequest(method, url, json) async def send(self, request, stream=False): self.calls.append((request.url, request.json_body)) return FakeResp_openai_gateway(status=500, payload={"error": "boom"}) async def aclose(self): pass class FakeBadRequestClient_openai_gateway: def __init__(self, *a, **k): self.calls = [] def build_request(self, method, url, headers=None, json=None, content=None): return FakeRequest(method, url, json) async def send(self, request, stream=False): self.calls.append((request.url, request.json_body)) return FakeResp_openai_gateway( status=400, payload={"error": {"message": "invalid request: bad param"}} ) async def aclose(self): pass def test_chat_does_not_fall_back_on_an_unrecoverable_bad_request(local_db, monkeypatch): from devplacepy.services.openai_gateway import routing monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeBadRequestClient_openai_gateway) routing.model_store.set( routing.ModelRouteIn(source_model="fb-badreq-backup", target_model="backup-target") ) routing.model_store.set( routing.ModelRouteIn( source_model="fb-badreq-primary", target_model="primary-target", fallback_model="fb-badreq-backup", ) ) try: svc = GatewayService() cfg = svc.effective_config() cfg["gateway_vision_enabled"] = False cfg["gateway_max_retries"] = 0 rt = svc.runtime() response = run_async( rt.handle_chat( {"model": "fb-badreq-primary", "messages": [{"role": "user", "content": "hi"}]}, cfg, ("guest", "no_fallback_on_bad_request"), "test", "default", ) ) assert response.status_code == 400 assert len(rt._client.calls) == 1 finally: routing.model_store.remove("fb-badreq-primary") routing.model_store.remove("fb-badreq-backup") def test_chat_falls_back_when_the_primary_model_fails(local_db, monkeypatch): from devplacepy.services.openai_gateway import routing monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeFallbackClient_openai_gateway) routing.model_store.set( routing.ModelRouteIn(source_model="fb-backup-route", target_model="backup-target") ) routing.model_store.set( routing.ModelRouteIn( source_model="fb-primary-route", target_model="primary-target", fallback_model="fb-backup-route", ) ) try: svc = GatewayService() cfg = svc.effective_config() cfg["gateway_vision_enabled"] = False cfg["gateway_max_retries"] = 0 rt = svc.runtime() response = run_async( rt.handle_chat( {"model": "fb-primary-route", "messages": [{"role": "user", "content": "hi"}]}, cfg, ("guest", "fallback_chat_success"), "test", "default", ) ) assert response.status_code == 200 assert rt._client.calls[0][1]["model"] == "primary-target" assert rt._client.calls[-1][1]["model"] == "backup-target" row = get_table("gateway_usage_ledger").find_one(owner_id="fallback_chat_success") assert row is not None assert row["requested_model"] == "fb-primary-route" assert row["model"] == "backup-target" assert row["success"] == 1 finally: routing.model_store.remove("fb-primary-route") routing.model_store.remove("fb-backup-route") class FakeChainClient_openai_gateway: def __init__(self, *a, **k): self.calls = [] def build_request(self, method, url, headers=None, json=None, content=None): return FakeRequest(method, url, json) async def send(self, request, stream=False): self.calls.append((request.url, request.json_body)) body = request.json_body or {} model = body.get("model") if model in ("chain-a-target", "chain-b-target"): return FakeResp_openai_gateway(status=500, payload={"error": "boom"}) return FakeResp_openai_gateway( payload={ "id": "x", "model": model, "choices": [{"message": {"content": "chain ok"}}], } ) async def aclose(self): pass def test_chat_walks_fallback_chain_past_failures(local_db, monkeypatch): from devplacepy.services.openai_gateway import routing monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeChainClient_openai_gateway) routing.model_store.set( routing.ModelRouteIn(source_model="chain-c", target_model="chain-c-target") ) routing.model_store.set( routing.ModelRouteIn( source_model="chain-b", target_model="chain-b-target", fallback_model="chain-c", ) ) routing.model_store.set( routing.ModelRouteIn( source_model="chain-a", target_model="chain-a-target", fallback_model="chain-b", ) ) try: svc = GatewayService() cfg = svc.effective_config() cfg["gateway_vision_enabled"] = False cfg["gateway_max_retries"] = 0 rt = svc.runtime() response = run_async( rt.handle_chat( {"model": "chain-a", "messages": [{"role": "user", "content": "hi"}]}, cfg, ("guest", "chain_walk"), "test", "default", ) ) assert response.status_code == 200 assert [c[1]["model"] for c in rt._client.calls] == [ "chain-a-target", "chain-b-target", "chain-c-target", ] row = get_table("gateway_usage_ledger").find_one(owner_id="chain_walk") assert row["model"] == "chain-c-target" assert row["fallback_used_route"] == "chain-c" finally: routing.model_store.remove("chain-a") routing.model_store.remove("chain-b") routing.model_store.remove("chain-c") def test_chat_preroutes_around_unhealthy_models(local_db, monkeypatch): from devplacepy.services.openai_gateway import health, routing monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeClient_openai_gateway) routing.model_store.set( routing.ModelRouteIn(source_model="pre-good", target_model="good-target") ) routing.model_store.set( routing.ModelRouteIn( source_model="pre-bad", target_model="bad-target", fallback_model="pre-good", ) ) health.record_health( "pre-bad", health.ProbeResult(False, "http_500", "boom", 1.0) ) try: svc = GatewayService() cfg = svc.effective_config() cfg["gateway_vision_enabled"] = False rt = svc.runtime() logged = [] response = run_async( rt.handle_chat( {"model": "pre-bad", "messages": [{"role": "user", "content": "hi"}]}, cfg, ("guest", "preroute"), "test", "default", log=logged.append, ) ) assert response.status_code == 200 assert len(rt._client.calls) == 1 assert rt._client.calls[0][1]["model"] == "good-target" assert any("is unhealthy, starting on" in line for line in logged) row = get_table("gateway_usage_ledger").find_one(owner_id="preroute") assert row["requested_model"] == "pre-bad" assert row["model"] == "good-target" finally: routing.model_store.remove("pre-bad") routing.model_store.remove("pre-good") def test_chat_tries_original_when_chain_is_exhausted(local_db, monkeypatch): from devplacepy.services.openai_gateway import health, routing monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeClient_openai_gateway) routing.model_store.set( routing.ModelRouteIn( source_model="exh-fb", target_model="exh-fb-target", fallback_model="exh-primary", ) ) routing.model_store.set( routing.ModelRouteIn( source_model="exh-primary", target_model="exh-primary-target", fallback_model="exh-fb", ) ) health.record_health( "exh-primary", health.ProbeResult(False, "http_500", "boom", 1.0) ) health.record_health( "exh-fb", health.ProbeResult(False, "http_500", "boom", 1.0) ) try: svc = GatewayService() cfg = svc.effective_config() cfg["gateway_vision_enabled"] = False rt = svc.runtime() response = run_async( rt.handle_chat( {"model": "exh-primary", "messages": [{"role": "user", "content": "hi"}]}, cfg, ("guest", "exhausted"), "test", "default", ) ) assert response.status_code == 200 assert [c[1]["model"] for c in rt._client.calls] == ["exh-primary-target"] finally: routing.model_store.remove("exh-primary") routing.model_store.remove("exh-fb") def test_models_endpoint_hides_unhealthy_routes(local_db): from devplacepy.services.openai_gateway import health, routing routing.model_store.set( routing.ModelRouteIn(source_model="hide-me", target_model="v/hide") ) health.record_health( "hide-me", health.ProbeResult(False, "http_500", "boom", 1.0) ) try: svc = GatewayService() payload = json.loads(svc._models_response().body.decode()) assert "hide-me" not in {m["id"] for m in payload["data"]} health.record_health( "hide-me", health.ProbeResult(True, "ok", "", 1.0) ) payload = json.loads(svc._models_response().body.decode()) assert "hide-me" in {m["id"] for m in payload["data"]} finally: routing.model_store.remove("hide-me") class FakeContextLengthClient_openai_gateway: def __init__(self, *a, **k): self.calls = [] def build_request(self, method, url, headers=None, json=None, content=None): return FakeRequest(method, url, json) async def send(self, request, stream=False): self.calls.append((request.url, request.json_body)) body = request.json_body or {} model = body.get("model") if model == "ctx-primary-target": return FakeResp_openai_gateway( status=400, payload={"error": {"message": "maximum context length exceeded"}}, ) return FakeResp_openai_gateway( payload={ "id": "x", "model": model, "choices": [{"message": {"content": "fallback ok"}}], } ) async def aclose(self): pass def test_chat_falls_back_on_a_context_length_error(local_db, monkeypatch): from devplacepy.services.openai_gateway import routing monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeContextLengthClient_openai_gateway) routing.model_store.set( routing.ModelRouteIn(source_model="ctx-backup-route", target_model="ctx-backup-target") ) routing.model_store.set( routing.ModelRouteIn( source_model="ctx-primary-route", target_model="ctx-primary-target", fallback_model="ctx-backup-route", ) ) try: svc = GatewayService() cfg = svc.effective_config() cfg["gateway_vision_enabled"] = False cfg["gateway_max_retries"] = 0 rt = svc.runtime() response = run_async( rt.handle_chat( {"model": "ctx-primary-route", "messages": [{"role": "user", "content": "hi"}]}, cfg, ("guest", "context_length_fallback_success"), "test", "default", ) ) assert response.status_code == 200 assert rt._client.calls[0][1]["model"] == "ctx-primary-target" assert rt._client.calls[-1][1]["model"] == "ctx-backup-target" row = get_table("gateway_usage_ledger").find_one( owner_id="context_length_fallback_success" ) assert row["success"] == 1 assert row["fallback_used_route"] == "ctx-backup-route" finally: routing.model_store.remove("ctx-primary-route") routing.model_store.remove("ctx-backup-route") def test_chat_skips_a_too_small_primary_and_goes_straight_to_fallback(local_db, monkeypatch): from devplacepy.services.openai_gateway import routing monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeContextLengthClient_openai_gateway) routing.model_store.set( routing.ModelRouteIn( source_model="ctx-precheck-backup", target_model="ctx-backup-target", context_window=1_000_000, ) ) routing.model_store.set( routing.ModelRouteIn( source_model="ctx-precheck-primary", target_model="ctx-primary-target", context_window=50, fallback_model="ctx-precheck-backup", ) ) try: svc = GatewayService() cfg = svc.effective_config() cfg["gateway_vision_enabled"] = False cfg["gateway_max_retries"] = 0 rt = svc.runtime() long_message = "word " * 2000 response = run_async( rt.handle_chat( { "model": "ctx-precheck-primary", "messages": [{"role": "user", "content": long_message}], }, cfg, ("guest", "context_precheck_skips_primary"), "test", "default", ) ) assert response.status_code == 200 # Only the fallback was ever called - the primary was known too small. assert len(rt._client.calls) == 1 assert rt._client.calls[0][1]["model"] == "ctx-backup-target" row = get_table("gateway_usage_ledger").find_one( owner_id="context_precheck_skips_primary" ) assert row["fallback_used_route"] == "ctx-precheck-backup" finally: routing.model_store.remove("ctx-precheck-primary") routing.model_store.remove("ctx-precheck-backup") def test_chat_falls_back_via_molodetz_when_the_client_sends_an_unrouted_model_name( local_db, monkeypatch ): from devplacepy.services.openai_gateway import routing original = routing.model_store.get("molodetz") routing.model_store.set( routing.ModelRouteIn(source_model="fb-molodetz-backup", target_model="backup-target") ) routing.model_store.set( routing.ModelRouteIn( source_model="molodetz", target_model="primary-target", fallback_model="fb-molodetz-backup", ) ) try: svc = GatewayService() cfg = svc.effective_config() cfg["gateway_vision_enabled"] = False cfg["gateway_max_retries"] = 0 cfg["gateway_force_model"] = False base_default_model = cfg["gateway_model"] class FakeUnroutedPrimaryFailsClient: def __init__(self, *a, **k): self.calls = [] def build_request(self, method, url, headers=None, json=None, content=None): return FakeRequest(method, url, json) async def send(self, request, stream=False): self.calls.append((request.url, request.json_body)) body = request.json_body or {} model = body.get("model") if model == base_default_model: return FakeResp_openai_gateway( status=402, payload={"error": {"message": "Insufficient Balance"}} ) return FakeResp_openai_gateway( payload={ "id": "x", "model": model, "choices": [{"message": {"content": "fallback ok"}}], } ) async def aclose(self): pass monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeUnroutedPrimaryFailsClient) rt = svc.runtime() # The client sends a model name that matches no configured route at all # (as a caller like dpc does, which is never told the exact "molodetz" # routing alias) - the primary call falls through to the flat/global # default model (no route overlay applies), but a failure there must # still consult molodetz's own configured fallback, not silently skip # it just because the client's literal string never matched anything. response = run_async( rt.handle_chat( { "model": "some-client-side-model-name", "messages": [{"role": "user", "content": "hi"}], }, cfg, ("guest", "fallback_chat_unrouted_client_model"), "test", "default", ) ) assert response.status_code == 200 assert rt._client.calls[0][1]["model"] == base_default_model assert rt._client.calls[-1][1]["model"] == "backup-target" row = get_table("gateway_usage_ledger").find_one( owner_id="fallback_chat_unrouted_client_model" ) assert row is not None assert row["requested_model"] == "some-client-side-model-name" assert row["model"] == "backup-target" assert row["success"] == 1 finally: routing.model_store.remove("fb-molodetz-backup") if original is not None: routing.model_store.set( routing.ModelRouteIn( **{ field: getattr(original, field) for field in routing.ModelRouteIn.model_fields } ) ) else: routing.model_store.remove("molodetz") def test_chat_returns_the_fallback_failure_when_both_models_fail(local_db, monkeypatch): from devplacepy.services.openai_gateway import routing monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeAlwaysFailClient_openai_gateway) routing.model_store.set( routing.ModelRouteIn(source_model="fb-backup-route2", target_model="backup-target2") ) routing.model_store.set( routing.ModelRouteIn( source_model="fb-primary-route2", target_model="primary-target2", fallback_model="fb-backup-route2", ) ) try: svc = GatewayService() cfg = svc.effective_config() cfg["gateway_vision_enabled"] = False cfg["gateway_max_retries"] = 0 rt = svc.runtime() response = run_async( rt.handle_chat( {"model": "fb-primary-route2", "messages": [{"role": "user", "content": "hi"}]}, cfg, ("guest", "fallback_chat_both_fail"), "test", "default", ) ) assert response.status_code == 500 assert len(rt._client.calls) == 2 assert rt._client.calls[0][1]["model"] == "primary-target2" assert rt._client.calls[1][1]["model"] == "backup-target2" finally: routing.model_store.remove("fb-primary-route2") routing.model_store.remove("fb-backup-route2") def test_chat_without_a_fallback_never_makes_a_second_call(local_db, monkeypatch): from devplacepy.services.openai_gateway import routing monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeAlwaysFailClient_openai_gateway) routing.model_store.set( routing.ModelRouteIn(source_model="fb-no-fallback", target_model="no-fallback-target") ) try: svc = GatewayService() cfg = svc.effective_config() cfg["gateway_vision_enabled"] = False cfg["gateway_max_retries"] = 0 rt = svc.runtime() run_async( rt.handle_chat( {"model": "fb-no-fallback", "messages": [{"role": "user", "content": "hi"}]}, cfg, ("guest", "no_fallback_configured"), "test", "default", ) ) assert len(rt._client.calls) == 1 finally: routing.model_store.remove("fb-no-fallback") def test_embeddings_unknown_model_falls_back_to_default(local_db, monkeypatch): monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeEmbedClient_openai_gateway) monkeypatch.setenv("OPENROUTER_API_KEY", "test-embed-key") svc = GatewayService() cfg = svc.effective_config() cfg["gateway_force_model"] = False cfg["gateway_embed_enabled"] = True cfg["gateway_embed_model"] = "qwen/qwen3-embedding-8b" rt = svc.runtime() run_async( rt.handle_embeddings( {"model": "nonexistent-embed-model", "input": "hello"}, cfg, ("guest", "fallback_test_embed"), "test", "default", ) ) assert rt._client.calls[-1][1]["model"] == "qwen/qwen3-embedding-8b" def test_images_unknown_model_falls_back_to_default(local_db, monkeypatch): monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeImageClient_openai_gateway) svc = GatewayService() cfg = svc.effective_config() cfg["gateway_force_model"] = False cfg["gateway_image_enabled"] = True cfg["gateway_image_model"] = "black-forest-labs/flux.2-pro" rt = svc.runtime() run_async( rt.handle_images( {"model": "nonexistent-image-model", "prompt": "test badge"}, cfg, ("guest", "fallback_test_img"), "test", "default", ) ) assert rt._client.calls[-1][1]["model"] == "black-forest-labs/flux.2-pro" class FakeClientWithUsage_openai_gateway(FakeClient_openai_gateway): async def send(self, request, stream=False): self.calls.append((request.url, request.json_body)) body = request.json_body or {} return FakeResp_openai_gateway( payload={ "id": "x", "model": body.get("model"), "choices": [{"message": {"content": "hi there"}}], "usage": {"prompt_tokens": 7, "completion_tokens": 3, "total_tokens": 10}, } ) def test_ollama_stream_does_not_send_stream_options(local_db, monkeypatch): monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeClient_openai_gateway) svc = GatewayService() cfg = svc.effective_config() cfg["gateway_upstream_url"] = "http://127.0.0.1:11434/api/chat" rt = svc.runtime() run_async( rt.handle_chat( { "messages": [{"role": "user", "content": "hi"}], "stream": True, "stream_options": {"include_usage": True}, }, cfg, ("guest", "ollama_stream_opts"), "test", "default", ) ) upstream_payload = rt._client.calls[-1][1] assert upstream_payload["stream"] is True assert "stream_options" not in upstream_payload assert upstream_payload["think"] is False def test_ollama_explicit_dialect_overrides_url(local_db, monkeypatch): monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeClient_openai_gateway) svc = GatewayService() cfg = svc.effective_config() cfg["gateway_upstream_url"] = "https://ai.example.com/v1/chat/completions" cfg["gateway_thinking_dialect"] = "ollama" rt = svc.runtime() run_async( rt.handle_chat( {"messages": [{"role": "user", "content": "hi"}]}, cfg, ("guest", "ollama_dialect_override"), "test", "default", ) ) body = rt._client.calls[-1][1] assert body["think"] is False assert "thinking" not in body def test_client_model_allowed_when_allow_client_model_on(local_db, monkeypatch): monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeClient_openai_gateway) svc = GatewayService() cfg = svc.effective_config() cfg["gateway_force_model"] = True cfg["gateway_allow_client_model"] = True cfg["gateway_model"] = "deepseek-chat" rt = svc.runtime() run_async( rt.handle_chat( {"model": "llama3.2", "messages": [{"role": "user", "content": "hi"}]}, cfg, ("guest", "client_model"), "test", "default", ) ) assert rt._client.calls[-1][1]["model"] == "llama3.2" def test_molodetz_alias_still_remapped_with_allow_client_model(local_db, monkeypatch): from devplacepy.services.openai_gateway import routing monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeClient_openai_gateway) original = routing.model_store.get("molodetz") routing.model_store.remove("molodetz") try: svc = GatewayService() cfg = svc.effective_config() cfg["gateway_force_model"] = True cfg["gateway_allow_client_model"] = True cfg["gateway_model"] = "deepseek-chat" rt = svc.runtime() run_async( rt.handle_chat( {"model": "molodetz", "messages": [{"role": "user", "content": "hi"}]}, cfg, ("guest", "molodetz_alias"), "test", "default", ) ) assert rt._client.calls[-1][1]["model"] == "deepseek-chat" finally: if original is not None: routing.model_store.set( routing.ModelRouteIn( **{ field: getattr(original, field) for field in routing.ModelRouteIn.model_fields } ) ) def test_stream_forwarded_to_upstream_with_forced_include_usage(local_db, monkeypatch): monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeClient_openai_gateway) svc = GatewayService() cfg = svc.effective_config() rt = svc.runtime() run_async( rt.handle_chat( { "messages": [{"role": "user", "content": "hi"}], "stream": True, "stream_options": {"include_usage": False}, }, cfg, ("guest", "test"), "test", "default", ) ) upstream_payload = rt._client.calls[-1][1] assert upstream_payload["stream"] is True assert upstream_payload["stream_options"] == {"include_usage": True} assert upstream_payload["thinking"] == {"type": "disabled"} assert "think" not in upstream_payload assert "reasoning" not in upstream_payload def test_non_stream_has_no_stream_options_upstream(local_db, monkeypatch): monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeClient_openai_gateway) svc = GatewayService() cfg = svc.effective_config() rt = svc.runtime() run_async( rt.handle_chat( { "messages": [{"role": "user", "content": "hi"}], "stream_options": {"include_usage": True}, }, cfg, ("guest", "test"), "test", "default", ) ) upstream_payload = rt._client.calls[-1][1] assert "stream_options" not in upstream_payload assert upstream_payload["stream"] is False def test_thinking_disabled_by_default(local_db, monkeypatch): monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeClient_openai_gateway) svc = GatewayService() cfg = svc.effective_config() rt = svc.runtime() run_async( rt.handle_chat( {"messages": [{"role": "user", "content": "hi"}]}, cfg, ("guest", "test"), "test", "default", ) ) body = rt._client.calls[-1][1] assert body["thinking"] == {"type": "disabled"} def test_client_think_true_enables_upstream_thinking(local_db, monkeypatch): monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeClient_openai_gateway) svc = GatewayService() cfg = svc.effective_config() rt = svc.runtime() run_async( rt.handle_chat( { "messages": [{"role": "user", "content": "hi"}], "think": True, }, cfg, ("guest", "test"), "test", "default", ) ) body = rt._client.calls[-1][1] assert body["thinking"] == {"type": "enabled"} assert "think" not in body def test_openrouter_upstream_gets_reasoning_none(local_db, monkeypatch): monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeClient_openai_gateway) svc = GatewayService() cfg = svc.effective_config() cfg["gateway_upstream_url"] = "https://openrouter.ai/api/v1/chat/completions" rt = svc.runtime() run_async( rt.handle_chat( {"messages": [{"role": "user", "content": "hi"}]}, cfg, ("guest", "test"), "test", "default", ) ) body = rt._client.calls[-1][1] assert body["reasoning"] == {"effort": "none"} assert "thinking" not in body def test_include_usage_emits_usage_chunk(local_db, monkeypatch): monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeClientWithUsage_openai_gateway) svc = GatewayService() cfg = svc.effective_config() rt = svc.runtime() resp = run_async( rt.handle_chat( { "messages": [{"role": "user", "content": "hi"}], "stream": True, "stream_options": {"include_usage": True}, }, cfg, ("guest", "test"), "test", "default", ) ) async def drain(): out = [] async for chunk in resp.body_iterator: out.append(chunk if isinstance(chunk, str) else chunk.decode()) return "".join(out) body = run_async(drain()) assert '"usage"' in body assert '"total_tokens": 10' in body assert "[DONE]" in body def test_no_usage_chunk_without_include_usage(local_db, monkeypatch): monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeClientWithUsage_openai_gateway) svc = GatewayService() cfg = svc.effective_config() rt = svc.runtime() resp = run_async( rt.handle_chat( {"messages": [{"role": "user", "content": "hi"}], "stream": True}, cfg, ("guest", "test"), "test", "default", ) ) async def drain(): out = [] async for chunk in resp.body_iterator: out.append(chunk if isinstance(chunk, str) else chunk.decode()) return "".join(out) body = run_async(drain()) assert '"usage"' not in body assert "[DONE]" in body def test_stream_headers_carry_no_cost_or_token_data(local_db, monkeypatch): monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeClientWithUsage_openai_gateway) svc = GatewayService() cfg = svc.effective_config() rt = svc.runtime() resp = run_async( rt.handle_chat( {"messages": [{"role": "user", "content": "hi"}], "stream": True}, cfg, ("guest", "test"), "test", "my-app", ) ) assert resp.headers["X-Gateway-Model"] == cfg["gateway_model"] assert resp.headers["X-Gateway-Backend"] == "chat" assert resp.headers["X-App-Reference"] == "my-app" assert "x-gateway-cost-usd" not in {k.lower() for k in resp.headers} assert "x-gateway-total-tokens" not in {k.lower() for k in resp.headers} def test_stream_records_ttft_and_inter_token_in_ledger(local_db, monkeypatch): monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeClientWithUsage_openai_gateway) svc = GatewayService() cfg = svc.effective_config() rt = svc.runtime() resp = run_async( rt.handle_chat( { "messages": [{"role": "user", "content": "hi"}], "stream": True, "stream_options": {"include_usage": True}, }, cfg, ("guest", "ttft_probe"), "test", "default", ) ) async def drain(): async for _ in resp.body_iterator: pass run_async(drain()) row = get_table("gateway_usage_ledger").find_one(owner_id="ttft_probe") assert row is not None assert row["backend"] == "chat" assert row["stream_requested"] == 1 assert row["success"] == 1 assert row["total_tokens"] == 10 assert row["ttft_ms"] is not None and row["ttft_ms"] >= 0.0 assert row["inter_token_ms"] is not None and row["inter_token_ms"] >= 0.0 class FakeClientReasoning_openai_gateway(FakeClient_openai_gateway): async def send(self, request, stream=False): self.calls.append((request.url, request.json_body)) body = request.json_body or {} return FakeResp_openai_gateway( payload={ "id": "x", "model": body.get("model"), "choices": [{"message": {"content": "x", "reasoning": "aaaaa"}}], "usage": {"prompt_tokens": 7, "completion_tokens": 3, "total_tokens": 10}, } ) def test_ollama_reasoning_delta_is_counted_as_a_content_chunk(local_db, monkeypatch): monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeClientReasoning_openai_gateway) svc = GatewayService() cfg = svc.effective_config() rt = svc.runtime() resp = run_async( rt.handle_chat( { "messages": [{"role": "user", "content": "hi"}], "stream": True, "stream_options": {"include_usage": True}, }, cfg, ("guest", "ollama_reasoning_probe"), "test", "default", ) ) async def drain(): async for _ in resp.body_iterator: pass run_async(drain()) row = get_table("gateway_usage_ledger").find_one(owner_id="ollama_reasoning_probe") assert row is not None assert row["inter_token_ms"] is not None and row["inter_token_ms"] >= 0.0 def test_stream_client_disconnect_records_failure_and_closes_upstream(local_db, monkeypatch): monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeClient_openai_gateway) svc = GatewayService() cfg = svc.effective_config() rt = svc.runtime() resp = run_async( rt.handle_chat( {"messages": [{"role": "user", "content": "hi"}], "stream": True}, cfg, ("guest", "disconnect_probe"), "test", "default", ) ) async def partial_then_close(): agen = resp.body_iterator await agen.__anext__() await agen.aclose() run_async(partial_then_close()) row = get_table("gateway_usage_ledger").find_one(owner_id="disconnect_probe") assert row is not None assert row["success"] == 0 assert row["error_category"] == "client_disconnected" def test_models_endpoint_publishes_molodetz(local_db): from devplacepy.services.openai_gateway import routing routing.seed_default_deepseek_routes() try: svc = GatewayService() resp = svc._models_response() payload = json.loads(resp.body.decode()) ids = {m["id"] for m in payload["data"]} assert payload["object"] == "list" assert {"molodetz", "molodetz-pro"} <= ids for model in payload["data"]: assert model["object"] == "model" assert model["owned_by"] == "molodetz" finally: routing.model_store.remove("molodetz") routing.model_store.remove("molodetz-pro") # Upstream header collision policy: only X-Gateway-Model is ever trusted and # forwarded from an upstream that happens to speak our own X-Gateway-* header # convention (e.g. another DevPlace-style gateway). Every other figure # (cost/tokens/latency/context/app-reference) must remain ours regardless of # what an upstream claims. See usage.upstream_reported_model / CLAUDE.md. class FakeClientUpstreamModelHeader_openai_gateway(FakeClient_openai_gateway): async def send(self, request, stream=False): self.calls.append((request.url, request.json_body)) body = request.json_body or {} return FakeResp_openai_gateway( payload={ "id": "x", "model": body.get("model"), "choices": [{"message": {"content": "hi there"}}], }, extra_headers={"X-Gateway-Model": "upstream/served-model-x"}, ) class FakeClientHostileModelHeader_openai_gateway(FakeClient_openai_gateway): async def send(self, request, stream=False): self.calls.append((request.url, request.json_body)) body = request.json_body or {} return FakeResp_openai_gateway( payload={ "id": "x", "model": body.get("model"), "choices": [{"message": {"content": "hi there"}}], }, extra_headers={"X-Gateway-Model": "evil\r\nX-Injected: true"}, ) class FakeEmbedClientUpstreamModelHeader_openai_gateway(FakeEmbedClient_openai_gateway): async def send(self, request, stream=False): self.calls.append((request.url, request.json_body)) body = request.json_body or {} return FakeResp_openai_gateway( payload={ "object": "list", "model": body.get("model"), "data": [{"object": "embedding", "index": 0, "embedding": [0.1, 0.2]}], "usage": {"prompt_tokens": 5, "total_tokens": 5}, }, extra_headers={"X-Gateway-Model": "upstream/embed-served"}, ) class FakeImageClientUpstreamModelHeader_openai_gateway(FakeImageClient_openai_gateway): async def send(self, request, stream=False): self.calls.append((request.url, request.json_body)) body = request.json_body or {} return FakeResp_openai_gateway( payload={ "created": 1, "model": body.get("model"), "data": [{"b64_json": "aGVsbG8="}], "usage": {"cost": 0.05}, }, extra_headers={"X-Gateway-Model": "upstream/image-served"}, ) class FakePassthroughClientUpstreamModelHeader_openai_gateway: def __init__(self, *a, **k): self.calls = [] def build_request(self, method, url, headers=None, json=None, content=None): return FakeRequest(method, url, json) async def send(self, request, stream=False): self.calls.append((request.url, request.json_body)) return FakeResp_openai_gateway( payload={"ok": True}, extra_headers={"X-Gateway-Model": "upstream/passthrough-served"}, ) async def aclose(self): pass def test_upstream_x_gateway_model_header_is_forwarded_for_chat(local_db, monkeypatch): monkeypatch.setattr( gwmod.httpx, "AsyncClient", FakeClientUpstreamModelHeader_openai_gateway ) svc = GatewayService() cfg = svc.effective_config() cfg["gateway_force_model"] = True cfg["gateway_model"] = "deepseek-chat" rt = svc.runtime() resp = run_async( rt.handle_chat( {"messages": [{"role": "user", "content": "hi"}]}, cfg, ("guest", "upstream_model_chat"), "test", "default", ) ) assert resp.headers["X-Gateway-Model"] == "upstream/served-model-x" row = get_table("gateway_usage_ledger").find_one(owner_id="upstream_model_chat") assert row is not None assert row["model"] == "deepseek-chat" def test_upstream_x_gateway_model_header_is_forwarded_for_streaming_chat( local_db, monkeypatch ): monkeypatch.setattr( gwmod.httpx, "AsyncClient", FakeClientUpstreamModelHeader_openai_gateway ) svc = GatewayService() cfg = svc.effective_config() rt = svc.runtime() resp = run_async( rt.handle_chat( {"messages": [{"role": "user", "content": "hi"}], "stream": True}, cfg, ("guest", "upstream_model_stream"), "test", "default", ) ) assert resp.headers["X-Gateway-Model"] == "upstream/served-model-x" assert resp.headers["X-Gateway-Backend"] == "chat" async def drain(): async for _ in resp.body_iterator: pass run_async(drain()) def test_upstream_x_gateway_model_header_is_forwarded_for_embeddings( local_db, monkeypatch ): monkeypatch.setattr( gwmod.httpx, "AsyncClient", FakeEmbedClientUpstreamModelHeader_openai_gateway ) monkeypatch.setenv("OPENROUTER_API_KEY", "test-embed-key") svc = GatewayService() cfg = svc.effective_config() cfg["gateway_embed_enabled"] = True cfg["gateway_embed_model"] = "qwen/qwen3-embedding-8b" rt = svc.runtime() resp = run_async( rt.handle_embeddings( {"input": "hello"}, cfg, ("guest", "upstream_model_embed"), "test", "default", ) ) assert resp.headers["X-Gateway-Model"] == "upstream/embed-served" row = get_table("gateway_usage_ledger").find_one(owner_id="upstream_model_embed") assert row is not None assert row["model"] == "qwen/qwen3-embedding-8b" def test_upstream_x_gateway_model_header_is_forwarded_for_images(local_db, monkeypatch): monkeypatch.setattr( gwmod.httpx, "AsyncClient", FakeImageClientUpstreamModelHeader_openai_gateway ) svc = GatewayService() cfg = svc.effective_config() cfg["gateway_image_enabled"] = True cfg["gateway_image_model"] = "black-forest-labs/flux.2-pro" rt = svc.runtime() resp = run_async( rt.handle_images( {"prompt": "badge"}, cfg, ("guest", "upstream_model_img"), "test", "default", ) ) assert resp.headers["X-Gateway-Model"] == "upstream/image-served" row = get_table("gateway_usage_ledger").find_one(owner_id="upstream_model_img") assert row is not None assert row["model"] == "black-forest-labs/flux.2-pro" def test_upstream_x_gateway_model_header_is_forwarded_for_passthrough( local_db, monkeypatch ): monkeypatch.setattr( gwmod.httpx, "AsyncClient", FakePassthroughClientUpstreamModelHeader_openai_gateway ) svc = GatewayService() cfg = svc.effective_config() cfg["gateway_model"] = "deepseek-chat" rt = svc.runtime() resp = run_async( rt.handle_passthrough( "POST", "responses", "application/json", b"{}", cfg, ("guest", "upstream_model_passthrough"), "test", "default", ) ) assert resp.headers["X-Gateway-Model"] == "upstream/passthrough-served" row = get_table("gateway_usage_ledger").find_one( owner_id="upstream_model_passthrough" ) assert row is not None assert row["model"] == "deepseek-chat" def test_no_upstream_model_header_keeps_our_own_model(local_db, monkeypatch): monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeClient_openai_gateway) svc = GatewayService() cfg = svc.effective_config() cfg["gateway_force_model"] = True cfg["gateway_model"] = "deepseek-chat" rt = svc.runtime() resp = run_async( rt.handle_chat( {"messages": [{"role": "user", "content": "hi"}]}, cfg, ("guest", "no_upstream_header"), "test", "default", ) ) assert resp.headers["X-Gateway-Model"] == "deepseek-chat" class FakeEmbeddedErrorClient_openai_gateway: """Mimics OpenRouter's documented behavior of answering 200 OK with the failure embedded in the JSON body (openrouter.ai/docs/api-reference/errors) instead of a non-2xx status.""" def __init__(self, *a, **k): self.calls = [] def build_request(self, method, url, headers=None, json=None, content=None): return FakeRequest(method, url, json) async def send(self, request, stream=False): self.calls.append((request.url, request.json_body)) body = request.json_body or {} model = body.get("model") if model == "primary-target": return FakeResp_openai_gateway( status=200, payload={"error": {"message": "no provider available", "code": 502}}, ) return FakeResp_openai_gateway( payload={ "id": "x", "model": model, "choices": [{"message": {"content": "fallback ok"}}], } ) async def aclose(self): pass def test_chat_falls_back_when_upstream_returns_200_with_an_embedded_error( local_db, monkeypatch ): from devplacepy.services.openai_gateway import routing monkeypatch.setattr( gwmod.httpx, "AsyncClient", FakeEmbeddedErrorClient_openai_gateway ) routing.model_store.set( routing.ModelRouteIn(source_model="eb-backup-route", target_model="backup-target") ) routing.model_store.set( routing.ModelRouteIn( source_model="eb-primary-route", target_model="primary-target", fallback_model="eb-backup-route", ) ) try: svc = GatewayService() cfg = svc.effective_config() cfg["gateway_vision_enabled"] = False cfg["gateway_max_retries"] = 0 rt = svc.runtime() response = run_async( rt.handle_chat( {"model": "eb-primary-route", "messages": [{"role": "user", "content": "hi"}]}, cfg, ("guest", "embedded_error_chat"), "test", "default", ) ) assert response.status_code == 200 assert rt._client.calls[0][1]["model"] == "primary-target" assert rt._client.calls[-1][1]["model"] == "backup-target" row = get_table("gateway_usage_ledger").find_one(owner_id="embedded_error_chat") assert row is not None assert row["requested_model"] == "eb-primary-route" assert row["model"] == "backup-target" assert row["success"] == 1 finally: routing.model_store.remove("eb-primary-route") routing.model_store.remove("eb-backup-route") class FakeAlwaysEmbeddedErrorClient_openai_gateway: def __init__(self, *a, **k): self.calls = [] def build_request(self, method, url, headers=None, json=None, content=None): return FakeRequest(method, url, json) async def send(self, request, stream=False): self.calls.append((request.url, request.json_body)) return FakeResp_openai_gateway( status=200, payload={"error": {"message": "no provider available", "code": 502}}, ) async def aclose(self): pass def test_chat_reports_502_when_upstream_returns_200_with_an_embedded_error_and_no_fallback( local_db, monkeypatch ): monkeypatch.setattr( gwmod.httpx, "AsyncClient", FakeAlwaysEmbeddedErrorClient_openai_gateway ) svc = GatewayService() cfg = svc.effective_config() cfg["gateway_vision_enabled"] = False cfg["gateway_max_retries"] = 0 rt = svc.runtime() response = run_async( rt.handle_chat( {"messages": [{"role": "user", "content": "hi"}]}, cfg, ("guest", "embedded_error_no_fallback"), "test", "default", ) ) assert response.status_code == 502 row = get_table("gateway_usage_ledger").find_one(owner_id="embedded_error_no_fallback") assert row is not None assert row["success"] == 0 assert row["error_category"] == "upstream_error" def test_ollama_dialect_sends_reasoning_effort_alongside_think(local_db, monkeypatch): # Ollama's native /api/chat honors a boolean `think`, but its # OpenAI-compatible /v1/chat/completions layer (what this gateway # actually calls) does not - it maps reasoning_effort/reasoning # instead (github.com/ollama/ollama issues #15288, #15293, #14820). monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeClient_openai_gateway) svc = GatewayService() cfg = svc.effective_config() cfg["gateway_upstream_url"] = "http://127.0.0.1:11434/v1/chat/completions" rt = svc.runtime() run_async( rt.handle_chat( {"messages": [{"role": "user", "content": "hi"}]}, cfg, ("guest", "ollama_reasoning_effort"), "test", "default", ) ) body = rt._client.calls[-1][1] assert body["think"] is False assert body["reasoning_effort"] == "none" def test_hostile_upstream_model_header_is_ignored_end_to_end(local_db, monkeypatch): monkeypatch.setattr( gwmod.httpx, "AsyncClient", FakeClientHostileModelHeader_openai_gateway ) svc = GatewayService() cfg = svc.effective_config() cfg["gateway_force_model"] = True cfg["gateway_model"] = "deepseek-chat" rt = svc.runtime() resp = run_async( rt.handle_chat( {"messages": [{"role": "user", "content": "hi"}]}, cfg, ("guest", "hostile_header"), "test", "default", ) ) assert resp.headers["X-Gateway-Model"] == "deepseek-chat" def test_bypass_preamble_skips_the_operator_preamble_when_allowed(local_db, monkeypatch): monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeClient_openai_gateway) svc = GatewayService() cfg = svc.effective_config() cfg["gateway_system_preamble"] = "OPERATOR-PREAMBLE-TEXT" rt = svc.runtime() run_async( rt.handle_chat( { "messages": [ {"role": "system", "content": "Reply with strict JSON only."}, {"role": "user", "content": "hi"}, ], "bypass_preamble": True, }, cfg, ("internal", "devii"), "test", "default", bypass_allowed=True, ) ) sent = rt._client.calls[-1][1] system_message = sent["messages"][0] assert system_message["role"] == "system" assert "OPERATOR-PREAMBLE-TEXT" not in system_message["content"] assert "Current date" in system_message["content"] assert "Reply with strict JSON only." in system_message["content"] assert "bypass_preamble" not in sent def test_bypass_preamble_ignored_without_internal_credentials(local_db, monkeypatch): monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeClient_openai_gateway) svc = GatewayService() cfg = svc.effective_config() cfg["gateway_system_preamble"] = "OPERATOR-PREAMBLE-TEXT" rt = svc.runtime() run_async( rt.handle_chat( {"messages": [{"role": "user", "content": "hi"}], "bypass_preamble": True}, cfg, ("user", "someuser"), "test", "default", bypass_allowed=False, ) ) sent = rt._client.calls[-1][1] system_message = sent["messages"][0] assert "OPERATOR-PREAMBLE-TEXT" in system_message["content"] assert "bypass_preamble" not in sent def test_bypass_preamble_keeps_the_clients_own_system_content(local_db, monkeypatch): monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeClient_openai_gateway) svc = GatewayService() cfg = svc.effective_config() cfg["gateway_system_preamble"] = "OPERATOR-PREAMBLE-TEXT" rt = svc.runtime() run_async( rt.handle_chat( { "messages": [ {"role": "system", "content": "Grade strictly. Reply JSON only."}, {"role": "user", "content": "hi"}, ], "bypass_preamble": True, }, cfg, ("internal", "devii"), "test", "default", bypass_allowed=True, ) ) sent = rt._client.calls[-1][1] system_message = sent["messages"][0] assert "OPERATOR-PREAMBLE-TEXT" not in system_message["content"] assert "Grade strictly. Reply JSON only." in system_message["content"] def test_internal_bypass_allowed_requires_the_exact_internal_key(local_db): set_setting("gateway_internal_key", "test-internal-secret") try: svc = GatewayService() cfg = svc.effective_config() assert ( svc.internal_bypass_allowed( _make_request_openai_gateway( headers={"X-Gateway-Internal-Key": "test-internal-secret"} ), cfg, ) is True ) assert ( svc.internal_bypass_allowed( _make_request_openai_gateway( headers={"X-Gateway-Internal-Key": "wrong-secret"} ), cfg, ) is False ) assert ( svc.internal_bypass_allowed( _make_request_openai_gateway( headers={"Authorization": "Bearer test-internal-secret"} ), cfg, ) is False ) assert svc.internal_bypass_allowed(_make_request_openai_gateway(), cfg) is False finally: set_setting("gateway_internal_key", "") class FakeResponsesResp_openai_gateway(FakeResp_openai_gateway): def __init__(self, status=200, payload=None, events=None, **kw): super().__init__(status=status, payload=payload, **kw) self._events = events or [] async def aiter_lines(self): for event in self._events: yield f"data: {json.dumps(event)}" yield "data: [DONE]" def _resp_obj_openai_gateway(model): return { "id": "resp_1", "object": "response", "created_at": 7, "status": "completed", "model": model, "output": [ { "type": "message", "role": "assistant", "content": [{"type": "output_text", "text": "hi there"}], } ], "usage": {"input_tokens": 5, "output_tokens": 7, "total_tokens": 12}, } def _resp_events_openai_gateway(model): return [ {"type": "response.created", "response": {"id": "resp_1"}}, {"type": "response.output_text.delta", "delta": "hi "}, {"type": "response.output_text.delta", "delta": "there"}, {"type": "response.completed", "response": _resp_obj_openai_gateway(model)}, ] def _resp_tool_obj_openai_gateway(model): obj = _resp_obj_openai_gateway(model) obj["output"].append( { "type": "function_call", "call_id": "call_7", "name": "ext_bash", "arguments": "{}", } ) return obj def _resp_tool_events_openai_gateway(model): return [ {"type": "response.created", "response": {"id": "resp_1"}}, { "type": "response.output_item.added", "output_index": 1, "item": { "type": "function_call", "call_id": "call_7", "name": "ext_bash", "arguments": "", }, }, { "type": "response.function_call_arguments.delta", "item_id": "call_7", "output_index": 1, "delta": "{}", }, { "type": "response.completed", "response": _resp_tool_obj_openai_gateway(model), }, ] class FakeResponsesClient_openai_gateway: behavior = "ok" def __init__(self, *a, **k): self.calls = [] self.last_headers = {} def build_request(self, method, url, headers=None, json=None, content=None): return FakeRequest(method, url, json, headers) async def send(self, request, stream=False): self.calls.append((request.url, request.json_body)) self.last_headers = dict(request.headers or {}) body = request.json_body or {} if self.behavior == "error-envelope": return FakeResp_openai_gateway( status=429, payload={ "error": { "message": "slow down", "type": "rate_limit", "code": "rate_limit_exceeded", } }, ) if self.behavior == "failed-object": return FakeResponsesResp_openai_gateway( payload={ "object": "response", "status": "failed", "error": {"message": "boom"}, } ) if self.behavior == "attestation-denied": return FakeResp_openai_gateway( status=403, payload={ "error": { "message": "This API key is billed to a Muse Code " "subscription. Muse Code subscriptions cover requests " "from the Muse Code client only.", "type": "permission_error", "code": "client_attestation_denied", } }, ) model = body.get("model", "") if self.behavior == "tool-call": return FakeResponsesResp_openai_gateway( payload=_resp_tool_obj_openai_gateway(model), events=_resp_tool_events_openai_gateway(model), ) return FakeResponsesResp_openai_gateway( payload=_resp_obj_openai_gateway(model), events=_resp_events_openai_gateway(model), ) async def post(self, url, headers=None, json=None, timeout=None): return await self.send(FakeRequest("POST", url, json), stream=False) async def aclose(self): pass def _setup_responses_route_openai_gateway(): from devplacepy.services.openai_gateway import routing routing.provider_store.set( routing.ProviderIn( name="gwmeta", base_url="https://api.meta.ai/v1/responses", api_key="sk-test", protocol="responses", ) ) routing.model_store.set( routing.ModelRouteIn( source_model="meta-chat", provider="gwmeta", target_model="muse-spark-1.3-contributor", kind="chat", ) ) def _teardown_responses_route_openai_gateway(): from devplacepy.services.openai_gateway import routing routing.model_store.remove("meta-chat") routing.provider_store.remove("gwmeta") def _chat_body_openai_gateway(**over): body = { "model": "meta-chat", "messages": [ {"role": "system", "content": "Be terse."}, {"role": "user", "content": "hi"}, ], } body.update(over) return body def test_responses_protocol_transpiles_request_and_response(local_db, monkeypatch): monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeResponsesClient_openai_gateway) FakeResponsesClient_openai_gateway.behavior = "ok" _setup_responses_route_openai_gateway() try: svc = GatewayService() cfg = svc.effective_config() cfg["gateway_vision_enabled"] = False rt = svc.runtime() resp = run_async( rt.handle_chat( _chat_body_openai_gateway(), cfg, ("guest", "test"), "test", "default" ) ) url, sent = rt._client.calls[-1] assert str(url) == "https://api.meta.ai/v1/responses" assert sent["model"] == "muse-spark-1.3-contributor" assert "messages" not in sent assert "Be terse." in sent["instructions"] assert sent["input"] == [ {"type": "message", "role": "user", "content": "hi"} ] assert sent["reasoning"] == {"effort": "minimal"} assert resp.status_code == 200 payload = json.loads(bytes(resp.body).decode()) assert payload["object"] == "chat.completion" assert payload["choices"][0]["message"]["content"] == "hi there" assert payload["choices"][0]["finish_reason"] == "stop" assert payload["usage"]["prompt_tokens"] == 5 assert resp.headers["X-Gateway-Model"] == "muse-spark-1.3-contributor" finally: _teardown_responses_route_openai_gateway() def test_muse_profile_headers_reach_the_wire(local_db, monkeypatch): from devplacepy.services.openai_gateway import routing monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeResponsesClient_openai_gateway) FakeResponsesClient_openai_gateway.behavior = "ok" routing.provider_store.set( routing.ProviderIn( name="wiremeta", base_url="https://api.meta.ai/v1/responses", api_key="sk-test", protocol="responses", client_profile="muse", ) ) routing.model_store.set( routing.ModelRouteIn( source_model="wire-muse", provider="wiremeta", target_model="muse-spark", kind="chat", ) ) try: svc = GatewayService() cfg = svc.effective_config() cfg["gateway_vision_enabled"] = False rt = svc.runtime() logged = [] resp = run_async( rt.handle_chat( {"model": "wire-muse", "messages": [{"role": "user", "content": "hi"}]}, cfg, ("guest", "test"), "test", "default", log=logged.append, ) ) assert resp.status_code == 200 sent_headers = rt._client.last_headers assert sent_headers["X-Client-Id"] == "tbh:exec" assert sent_headers["User-Agent"].startswith("muse-build/") session = sent_headers["X-Tbh-Session-Id"] assert session assert sent_headers["X-Meta-Ai-Gateway-Session-Id"] == session assert sent_headers["X-Tbh-Thread-Id"] == session assert sent_headers["X-Tbh-Root-Session-Id"] == session assert any("(client profile 'muse')" in line for line in logged) finally: routing.model_store.remove("wire-muse") routing.provider_store.remove("wiremeta") def _bash_tool_body_openai_gateway(**over): body = _chat_body_openai_gateway(**over) body["tools"] = [ { "type": "function", "function": { "name": "bash", "description": "run shell", "parameters": {"type": "object", "properties": {}}, }, } ] return body def test_meta_tool_names_aliased_on_wire_and_restored(local_db, monkeypatch): monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeResponsesClient_openai_gateway) FakeResponsesClient_openai_gateway.behavior = "tool-call" _setup_responses_route_openai_gateway() try: svc = GatewayService() cfg = svc.effective_config() cfg["gateway_vision_enabled"] = False rt = svc.runtime() logged = [] resp = run_async( rt.handle_chat( _bash_tool_body_openai_gateway(), cfg, ("guest", "test"), "test", "default", log=logged.append, ) ) _, sent = rt._client.calls[-1] assert sent["tools"][0]["name"] == "ext_bash" assert any("bash->ext_bash" in line for line in logged) assert resp.status_code == 200 payload = json.loads(bytes(resp.body).decode()) calls = payload["choices"][0]["message"]["tool_calls"] assert calls[0]["function"]["name"] == "bash" assert payload["choices"][0]["finish_reason"] == "tool_calls" finally: FakeResponsesClient_openai_gateway.behavior = "ok" _teardown_responses_route_openai_gateway() def test_meta_tool_names_restored_on_stream_chunks(local_db, monkeypatch): monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeResponsesClient_openai_gateway) FakeResponsesClient_openai_gateway.behavior = "tool-call" _setup_responses_route_openai_gateway() try: svc = GatewayService() cfg = svc.effective_config() cfg["gateway_vision_enabled"] = False rt = svc.runtime() resp = run_async( rt.handle_chat( _bash_tool_body_openai_gateway(stream=True), cfg, ("guest", "test"), "test", "default", ) ) async def drain(): out = [] async for chunk in resp.body_iterator: out.append(chunk if isinstance(chunk, str) else chunk.decode()) return "".join(out) body = run_async(drain()) assert '"name": "bash"' in body assert "ext_bash" not in body assert "[DONE]" in body finally: FakeResponsesClient_openai_gateway.behavior = "ok" _teardown_responses_route_openai_gateway() def _setup_chat_meta_route_openai_gateway(): from devplacepy.services.openai_gateway import routing routing.provider_store.set( routing.ProviderIn( name="gwmetacc", base_url="https://api.meta.ai/v1/chat/completions", api_key="sk-test", ) ) routing.model_store.set( routing.ModelRouteIn( source_model="meta-chat-cc", provider="gwmetacc", target_model="muse-spark", kind="chat", ) ) def _teardown_chat_meta_route_openai_gateway(): from devplacepy.services.openai_gateway import routing routing.model_store.remove("meta-chat-cc") routing.provider_store.remove("gwmetacc") def _tai_tool_body_openai_gateway(**over): # Mirrors a real tai/detz turn: core tools are always sent, so "shell" # (a Pi-verified gate trigger missing from the denylist until # 2026-11-10) rides every request alongside the lazy "read_file". body = _chat_body_openai_gateway(**over) body["model"] = "meta-chat-cc" body["tools"] = [ { "type": "function", "function": { "name": "shell", "description": "run a shell command", "parameters": {"type": "object", "properties": {}}, }, }, { "type": "function", "function": { "name": "read_file", "description": "read a file", "parameters": {"type": "object", "properties": {}}, }, }, { "type": "function", "function": { "name": "web_fetch", "description": "fetch a URL", "parameters": {"type": "object", "properties": {}}, }, }, ] return body class FakeChatMetaStreamResp_openai_gateway(FakeResp_openai_gateway): def __init__(self, payload): super().__init__(status=200, payload=payload) self._payload = payload async def aiter_lines(self): calls = self._payload["choices"][0]["message"]["tool_calls"] yield ( 'data: {"id":"chatcmpl-x","object":"chat.completion.chunk",' '"model":"m","choices":[{"index":0,"delta":{"role":"assistant"}}]}' ) for pos, call in enumerate(calls): name = call["function"]["name"] yield ( 'data: {"id":"chatcmpl-x","object":"chat.completion.chunk",' '"model":"m","choices":[{"index":0,"delta":{"tool_calls":' '[{"index":%d,"id":"call_%d","function":{"name":"%s",' '"arguments":"{}"}}]}}]}' % (pos, pos, name) ) yield ( 'data: {"id":"chatcmpl-x","object":"chat.completion.chunk",' '"model":"m","choices":[{"index":0,"delta":{},' '"finish_reason":"tool_calls"}]}' ) yield "data: [DONE]" class FakeChatMetaClient_openai_gateway: def __init__(self, *a, **k): self.calls = [] def build_request(self, method, url, headers=None, json=None, content=None): return FakeRequest(method, url, json, headers) async def send(self, request, stream=False): self.calls.append((request.url, request.json_body)) body = request.json_body or {} wire_names = [ (tool.get("function") or {}).get("name") for tool in body.get("tools") or [] ] payload = { "id": "chatcmpl-x", "object": "chat.completion", "model": body.get("model"), "choices": [ { "index": 0, "message": { "role": "assistant", "content": "", "tool_calls": [ { "id": "call_7", "type": "function", "function": {"name": name, "arguments": "{}"}, } for name in wire_names ], }, "finish_reason": "tool_calls", } ], "usage": {"prompt_tokens": 5, "completion_tokens": 2, "total_tokens": 7}, } if stream: return FakeChatMetaStreamResp_openai_gateway(payload) return FakeResp_openai_gateway( payload=payload, content=json.dumps(payload).encode() ) async def post(self, url, headers=None, json=None, timeout=None): return await self.send(FakeRequest("POST", url, json), stream=False) async def aclose(self): pass def test_meta_tool_names_aliased_on_chat_completions_wire(local_db, monkeypatch): monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeChatMetaClient_openai_gateway) _setup_chat_meta_route_openai_gateway() try: svc = GatewayService() cfg = svc.effective_config() cfg["gateway_vision_enabled"] = False rt = svc.runtime() logged = [] resp = run_async( rt.handle_chat( _tai_tool_body_openai_gateway(), cfg, ("guest", "test"), "test", "default", log=logged.append, ) ) _, sent = rt._client.calls[-1] wire = [tool["function"]["name"] for tool in sent["tools"]] assert wire == ["ext_shell", "ext_read_file", "ext_web_fetch"] assert any("shell->ext_shell" in line for line in logged) assert any("read_file->ext_read_file" in line for line in logged) assert any("web_fetch->ext_web_fetch" in line for line in logged) assert resp.status_code == 200 payload = json.loads(bytes(resp.body).decode()) calls = payload["choices"][0]["message"]["tool_calls"] assert [call["function"]["name"] for call in calls] == [ "shell", "read_file", "web_fetch", ] assert payload["choices"][0]["finish_reason"] == "tool_calls" finally: _teardown_chat_meta_route_openai_gateway() def test_meta_tool_names_restored_on_chat_completions_stream( local_db, monkeypatch ): monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeChatMetaClient_openai_gateway) _setup_chat_meta_route_openai_gateway() try: svc = GatewayService() cfg = svc.effective_config() cfg["gateway_vision_enabled"] = False rt = svc.runtime() resp = run_async( rt.handle_chat( _tai_tool_body_openai_gateway(stream=True), cfg, ("guest", "test"), "test", "default", ) ) async def drain(): out = [] async for chunk in resp.body_iterator: out.append(chunk if isinstance(chunk, str) else chunk.decode()) return "".join(out) body = run_async(drain()) _, sent = rt._client.calls[-1] wire = [tool["function"]["name"] for tool in sent["tools"]] assert wire == ["ext_shell", "ext_read_file", "ext_web_fetch"] assert '"name": "shell"' in body assert '"name": "read_file"' in body assert '"name": "web_fetch"' in body assert "ext_shell" not in body assert "ext_read_file" not in body assert "ext_web_fetch" not in body assert "[DONE]" in body finally: _teardown_chat_meta_route_openai_gateway() def test_responses_protocol_stream_relays_chat_chunks(local_db, monkeypatch): monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeResponsesClient_openai_gateway) FakeResponsesClient_openai_gateway.behavior = "ok" _setup_responses_route_openai_gateway() try: svc = GatewayService() cfg = svc.effective_config() cfg["gateway_vision_enabled"] = False rt = svc.runtime() resp = run_async( rt.handle_chat( _chat_body_openai_gateway(stream=True), cfg, ("guest", "test"), "test", "default", ) ) async def drain(): out = [] async for chunk in resp.body_iterator: out.append(chunk if isinstance(chunk, str) else chunk.decode()) return "".join(out) body = run_async(drain()) assert "chat.completion.chunk" in body assert "hi " in body and "there" in body assert '"usage"' not in body assert "[DONE]" in body finally: _teardown_responses_route_openai_gateway() def test_responses_protocol_stream_relays_usage_when_asked(local_db, monkeypatch): monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeResponsesClient_openai_gateway) FakeResponsesClient_openai_gateway.behavior = "ok" _setup_responses_route_openai_gateway() try: svc = GatewayService() cfg = svc.effective_config() cfg["gateway_vision_enabled"] = False rt = svc.runtime() resp = run_async( rt.handle_chat( _chat_body_openai_gateway( stream=True, stream_options={"include_usage": True} ), cfg, ("guest", "test"), "test", "default", ) ) _, sent = rt._client.calls[-1] assert "stream_options" not in sent async def drain(): out = [] async for chunk in resp.body_iterator: out.append(chunk if isinstance(chunk, str) else chunk.decode()) return "".join(out) body = run_async(drain()) assert '"usage"' in body assert '"prompt_tokens": 5' in body usage_chunks = [ json.loads(line[len("data:"):]) for line in body.splitlines() if line.startswith("data: {") and '"usage"' in line ] assert len(usage_chunks) == 1 assert usage_chunks[0]["choices"] == [] finally: _teardown_responses_route_openai_gateway() def test_responses_protocol_relays_error_envelope(local_db, monkeypatch): monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeResponsesClient_openai_gateway) FakeResponsesClient_openai_gateway.behavior = "error-envelope" _setup_responses_route_openai_gateway() try: svc = GatewayService() cfg = svc.effective_config() cfg["gateway_vision_enabled"] = False rt = svc.runtime() resp = run_async( rt.handle_chat( _chat_body_openai_gateway(), cfg, ("guest", "test"), "test", "default" ) ) assert resp.status_code == 429 payload = json.loads(bytes(resp.body).decode()) assert payload["error"]["message"] == "slow down" assert payload["error"]["code"] == "rate_limit_exceeded" finally: FakeResponsesClient_openai_gateway.behavior = "ok" _teardown_responses_route_openai_gateway() def test_responses_protocol_attestation_denied_logs_guidance( local_db, monkeypatch ): monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeResponsesClient_openai_gateway) FakeResponsesClient_openai_gateway.behavior = "attestation-denied" _setup_responses_route_openai_gateway() try: svc = GatewayService() cfg = svc.effective_config() cfg["gateway_vision_enabled"] = False rt = svc.runtime() logged = [] resp = run_async( rt.handle_chat( _chat_body_openai_gateway(), cfg, ("guest", "test"), "test", "default", log=logged.append, ) ) assert resp.status_code == 403 payload = json.loads(bytes(resp.body).decode()) assert payload["error"]["code"] == "client_attestation_denied" assert any("pay-as-you-go" in line for line in logged) finally: FakeResponsesClient_openai_gateway.behavior = "ok" _teardown_responses_route_openai_gateway() def test_upstream_auth_guidance_covers_401_and_403(): denied = FakeResp_openai_gateway( status=403, payload={ "error": { "message": "subscription only", "type": "permission_error", "code": "client_attestation_denied", } }, ) guidance = gwmod._upstream_auth_guidance(denied, denied.json()) assert "pay-as-you-go" in guidance assert "aliasing" in guidance assert "MUSE_RESERVED_TOOL_NAMES" in guidance other_denied = FakeResp_openai_gateway(status=403, payload={"error": "nope"}) assert ( gwmod._upstream_auth_guidance(other_denied, other_denied.json()) is None ) unauthed = FakeResp_openai_gateway( status=401, payload={"error": {"message": "User not found.", "code": 401}} ) guidance = gwmod._upstream_auth_guidance(unauthed, unauthed.json()) assert "api_key" in guidance assert "User not found." in guidance server_error = FakeResp_openai_gateway(status=500, payload={"error": "x"}) assert ( gwmod._upstream_auth_guidance(server_error, server_error.json()) is None ) def test_should_fallback_model_unavailable_but_not_bad_request(): unavailable = FakeResp_openai_gateway( status=400, payload={"error": {"message": "Model is unavailable."}}, ) assert ( gwmod._should_fallback(unavailable, None, {"circuit_open": False}) is True ) malformed = FakeResp_openai_gateway( status=400, payload={"error": "bad schema"} ) assert ( gwmod._should_fallback(malformed, None, {"circuit_open": False}) is False ) def test_responses_protocol_failed_object_is_502(local_db, monkeypatch): monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeResponsesClient_openai_gateway) FakeResponsesClient_openai_gateway.behavior = "failed-object" _setup_responses_route_openai_gateway() try: svc = GatewayService() cfg = svc.effective_config() cfg["gateway_vision_enabled"] = False rt = svc.runtime() resp = run_async( rt.handle_chat( _chat_body_openai_gateway(), cfg, ("guest", "test"), "test", "default" ) ) assert resp.status_code == 502 finally: FakeResponsesClient_openai_gateway.behavior = "ok" _teardown_responses_route_openai_gateway() def test_responses_protocol_client_reasoning_effort_passes_through( local_db, monkeypatch ): monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeResponsesClient_openai_gateway) FakeResponsesClient_openai_gateway.behavior = "ok" _setup_responses_route_openai_gateway() try: svc = GatewayService() cfg = svc.effective_config() cfg["gateway_vision_enabled"] = False rt = svc.runtime() run_async( rt.handle_chat( _chat_body_openai_gateway(reasoning_effort="high"), cfg, ("guest", "test"), "test", "default", ) ) _, sent = rt._client.calls[-1] assert sent["reasoning"] == {"effort": "high"} finally: _teardown_responses_route_openai_gateway() def test_responses_protocol_default_provider(local_db, monkeypatch): monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeResponsesClient_openai_gateway) FakeResponsesClient_openai_gateway.behavior = "ok" svc = GatewayService() cfg = svc.effective_config() cfg["gateway_vision_enabled"] = False cfg["gateway_upstream_url"] = "https://api.meta.ai/v1/responses" cfg["gateway_upstream_protocol"] = "responses" cfg["gateway_model"] = "muse-spark-1.3-contributor" rt = svc.runtime() resp = run_async( rt.handle_chat( {"messages": [{"role": "user", "content": "hi"}]}, cfg, ("guest", "test"), "test", "default", ) ) url, sent = rt._client.calls[-1] assert str(url) == "https://api.meta.ai/v1/responses" assert sent["input"] == [{"type": "message", "role": "user", "content": "hi"}] assert resp.status_code == 200 payload = json.loads(bytes(resp.body).decode()) assert payload["object"] == "chat.completion" EMULATED_CALL_TEXT_openai_gateway = ( "Sure.\n\n" '{"name": "bash", "arguments": {"command": "ls"}}\n' "" ) BASH_TOOL_openai_gateway = { "type": "function", "function": { "name": "bash", "description": "Run a shell command", "parameters": { "type": "object", "properties": {"command": {"type": "string"}}, "required": ["command"], }, }, } def _emulated_obj_openai_gateway(model, text): obj = _resp_obj_openai_gateway(model) obj["output"][0]["content"][0]["text"] = text return obj class FakeEmulatedClient_openai_gateway(FakeResponsesClient_openai_gateway): text = EMULATED_CALL_TEXT_openai_gateway async def send(self, request, stream=False): self.calls.append((request.url, request.json_body)) model = (request.json_body or {}).get("model", "") obj = _emulated_obj_openai_gateway(model, self.text) events = [{"type": "response.created", "response": {"id": "resp_1"}}] for i in range(0, len(self.text), 4): events.append( {"type": "response.output_text.delta", "delta": self.text[i : i + 4]} ) events.append({"type": "response.completed", "response": obj}) return FakeResponsesResp_openai_gateway(payload=obj, events=events) def _setup_emulated_route_openai_gateway(): from devplacepy.services.openai_gateway import routing routing.provider_store.set( routing.ProviderIn( name="gwmeta", base_url="https://api.meta.ai/v1/responses", api_key="sk-test", protocol="responses", ) ) routing.model_store.set( routing.ModelRouteIn( source_model="meta-chat", provider="gwmeta", target_model="muse-spark-1.3-contributor", kind="chat", tools_mode="emulated", ) ) def _emulated_chat_openai_gateway(monkeypatch, body, **tools): monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeEmulatedClient_openai_gateway) _setup_emulated_route_openai_gateway() svc = GatewayService() cfg = svc.effective_config() cfg["gateway_vision_enabled"] = False rt = svc.runtime() resp = run_async(rt.handle_chat(body, cfg, ("guest", "test"), "test", "default")) return rt, resp def test_emulated_tools_meta_responses_nonstream(local_db, monkeypatch): body = _chat_body_openai_gateway(tools=[BASH_TOOL_openai_gateway], tool_choice="auto") try: rt, resp = _emulated_chat_openai_gateway(monkeypatch, body) url, sent = rt._client.calls[-1] assert str(url) == "https://api.meta.ai/v1/responses" assert "tools" not in sent and "tool_choice" not in sent assert "" in sent["instructions"] and '"bash"' in sent["instructions"] assert resp.status_code == 200 message = json.loads(bytes(resp.body).decode())["choices"][0] assert message["finish_reason"] == "tool_calls" call = message["message"]["tool_calls"][0] assert call["function"]["name"] == "bash" assert json.loads(call["function"]["arguments"]) == {"command": "ls"} assert message["message"]["content"] == "Sure." finally: _teardown_responses_route_openai_gateway() def test_emulated_tools_meta_responses_stream(local_db, monkeypatch): body = _chat_body_openai_gateway( stream=True, tools=[BASH_TOOL_openai_gateway] ) try: rt, resp = _emulated_chat_openai_gateway(monkeypatch, body) async def drain(): out = [] async for chunk in resp.body_iterator: out.append(chunk if isinstance(chunk, str) else chunk.decode()) return "".join(out) raw = run_async(drain()) chunks = [ json.loads(line[5:]) for line in raw.splitlines() if line.startswith("data:") and "[DONE]" not in line ] text = "".join( c["choices"][0]["delta"].get("content", "") for c in chunks if c.get("choices") ) calls = [ tc for c in chunks for choice in c.get("choices") or [] for tc in choice["delta"].get("tool_calls") or [] ] assert "" not in text and text.strip() == "Sure." assert len(calls) == 1 and calls[0]["function"]["name"] == "bash" finishes = [ c["choices"][0]["finish_reason"] for c in chunks if c.get("choices") and c["choices"][0].get("finish_reason") ] assert finishes[-1] == "tool_calls" assert raw.rstrip().endswith("[DONE]") finally: _teardown_responses_route_openai_gateway() def test_emulated_tools_convert_history_for_the_meta_transpiler(local_db, monkeypatch): history = [ {"role": "user", "content": "go"}, { "role": "assistant", "content": None, "tool_calls": [ { "id": "call_1", "type": "function", "function": {"name": "bash", "arguments": '{"command": "ls"}'}, } ], }, {"role": "tool", "tool_call_id": "call_1", "content": "a.txt"}, ] body = _chat_body_openai_gateway(messages=history, tools=[BASH_TOOL_openai_gateway]) try: rt, resp = _emulated_chat_openai_gateway(monkeypatch, body) _, sent = rt._client.calls[-1] assert resp.status_code == 200 assert all(item.get("type") == "message" for item in sent["input"]) joined = json.dumps(sent["input"]) assert "" in joined and "" in joined and "a.txt" in joined assert "function_call" not in joined finally: _teardown_responses_route_openai_gateway() def test_emulated_tools_serve_large_catalogs_with_no_cap(local_db, monkeypatch): tools = [ { "type": "function", "function": { "name": f"tool_{index}", "description": f"tool number {index}", "parameters": {"type": "object", "properties": {}, "required": []}, }, } for index in range(200) ] body = _chat_body_openai_gateway(tools=tools, tool_choice="auto") try: rt, resp = _emulated_chat_openai_gateway(monkeypatch, body) assert resp.status_code == 200 assert len(rt._client.calls) == 1 _, sent = rt._client.calls[-1] assert "tools" not in sent and "tool_choice" not in sent assert '"name": "tool_0"' in sent["instructions"] assert '"name": "tool_199"' in sent["instructions"] assert len(rt._catalog_cache._entries) == 1 finally: _teardown_responses_route_openai_gateway() def test_native_route_keeps_forwarding_tools(local_db, monkeypatch): monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeResponsesClient_openai_gateway) FakeResponsesClient_openai_gateway.behavior = "ok" _setup_responses_route_openai_gateway() try: svc = GatewayService() cfg = svc.effective_config() cfg["gateway_vision_enabled"] = False rt = svc.runtime() run_async( rt.handle_chat( _chat_body_openai_gateway(tools=[BASH_TOOL_openai_gateway]), cfg, ("guest", "test"), "test", "default", ) ) _, sent = rt._client.calls[-1] assert "tools" in sent assert "" not in sent["instructions"] finally: _teardown_responses_route_openai_gateway() VAIDS_REJECTION_openai_gateway = { "details": {"_errors": ["Unrecognized key(s) in object: 'thinking'"]}, "error": "Invalid request parameters", "issues": [ { "code": "unrecognized_keys", "keys": ["thinking"], "path": [], "message": "Unrecognized key(s) in object: 'thinking'", } ], } def _rejection_openai_gateway(key): return { "details": {"_errors": [f"Unrecognized key(s) in object: '{key}'"]}, "error": "Invalid request parameters", "issues": [{"code": "unrecognized_keys", "keys": [key], "path": []}], } class FakeStrictClient_openai_gateway: rejected = ("thinking",) def __init__(self, *a, **k): self.calls = [] def build_request(self, method, url, headers=None, json=None, content=None): return FakeRequest(method, url, json) async def send(self, request, stream=False): body = request.json_body or {} self.calls.append((request.url, dict(body))) for key in self.rejected: if key in body: return FakeResp_openai_gateway( status=400, payload=_rejection_openai_gateway(key) ) return FakeResp_openai_gateway( payload={ "id": "x", "model": body.get("model"), "choices": [{"message": {"content": "hi there"}}], } ) async def aclose(self): pass class FakeStubbornClient_openai_gateway(FakeStrictClient_openai_gateway): async def send(self, request, stream=False): self.calls.append((request.url, dict(request.json_body or {}))) return FakeResp_openai_gateway( status=400, payload=_rejection_openai_gateway("thinking") ) class FakeReasoningStrictClient_openai_gateway(FakeStrictClient_openai_gateway): rejected = ("reasoning",) def _shape_cfg_openai_gateway(): svc = GatewayService() cfg = svc.effective_config() cfg["gateway_vision_enabled"] = False cfg["gateway_max_retries"] = 0 return svc, cfg def _chat_openai_gateway(rt, cfg, owner, stream=False): body = {"messages": [{"role": "user", "content": "hi"}]} if stream: body["stream"] = True return run_async(rt.handle_chat(body, cfg, ("guest", owner), "test", "default")) def test_chat_auto_resolves_a_rejected_thinking_key_and_remembers_it(local_db, monkeypatch): from devplacepy.services.openai_gateway.learned import learned_store monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeStrictClient_openai_gateway) svc, cfg = _shape_cfg_openai_gateway() model = cfg["gateway_model"] learned_store.forget("default", model) try: rt = svc.runtime() response = _chat_openai_gateway(rt, cfg, "shape_resolve_first") assert response.status_code == 200 assert len(rt._client.calls) == 2 assert "thinking" in rt._client.calls[0][1] assert "thinking" not in rt._client.calls[1][1] learned = learned_store.get("default", model) assert learned is not None and learned.thinking_dialect == "none" row = get_table("gateway_usage_ledger").find_one(owner_id="shape_resolve_first") assert row["resolved_method"] == "none" assert row["success"] == 1 second = _chat_openai_gateway(rt, cfg, "shape_resolve_second") assert second.status_code == 200 assert len(rt._client.calls) == 3 assert "thinking" not in rt._client.calls[2][1] finally: learned_store.forget("default", model) def test_chat_auto_resolve_works_for_streams(local_db, monkeypatch): from devplacepy.services.openai_gateway.learned import learned_store monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeStrictClient_openai_gateway) svc, cfg = _shape_cfg_openai_gateway() model = cfg["gateway_model"] learned_store.forget("default", model) try: rt = svc.runtime() response = _chat_openai_gateway(rt, cfg, "shape_resolve_stream", stream=True) async def drain(): out = [] async for chunk in response.body_iterator: out.append(chunk if isinstance(chunk, str) else chunk.decode()) return "".join(out) assert "[DONE]" in run_async(drain()) assert len(rt._client.calls) == 2 assert learned_store.get("default", model) is not None finally: learned_store.forget("default", model) def test_chat_genuine_bad_request_is_not_auto_resolved(local_db, monkeypatch): from devplacepy.services.openai_gateway.learned import learned_store monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeBadRequestClient_openai_gateway) svc, cfg = _shape_cfg_openai_gateway() model = cfg["gateway_model"] learned_store.forget("default", model) rt = svc.runtime() response = _chat_openai_gateway(rt, cfg, "shape_genuine_bad") assert response.status_code == 400 assert len(rt._client.calls) == 1 assert learned_store.get("default", model) is None def test_chat_exhausted_resolution_returns_the_original_error_and_stores_nothing( local_db, monkeypatch ): from devplacepy.services.openai_gateway.learned import learned_store from devplacepy.services.openai_gateway.shape import MAX_RESOLVE_ATTEMPTS monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeStubbornClient_openai_gateway) svc, cfg = _shape_cfg_openai_gateway() model = cfg["gateway_model"] learned_store.forget("default", model) rt = svc.runtime() response = _chat_openai_gateway(rt, cfg, "shape_exhausted") assert response.status_code == 400 assert "thinking" in response.body.decode() assert len(rt._client.calls) <= 1 + MAX_RESOLVE_ATTEMPTS assert learned_store.get("default", model) is None def test_chat_rejected_learned_method_is_forgotten_and_ladder_restarts(local_db, monkeypatch): from devplacepy.services.openai_gateway.learned import learned_store from devplacepy.services.openai_gateway.shape import ShapeMethod monkeypatch.setattr( gwmod.httpx, "AsyncClient", FakeReasoningStrictClient_openai_gateway ) svc, cfg = _shape_cfg_openai_gateway() model = cfg["gateway_model"] learned_store.remember("default", model, ShapeMethod("openrouter")) try: rt = svc.runtime() response = _chat_openai_gateway(rt, cfg, "shape_heal") assert response.status_code == 200 assert "reasoning" in rt._client.calls[0][1] assert "thinking" in rt._client.calls[1][1] assert learned_store.get("default", model) is None finally: learned_store.forget("default", model) def test_fallback_attempt_is_shaped_with_its_own_learned_method(local_db, monkeypatch): from devplacepy.services.openai_gateway import routing from devplacepy.services.openai_gateway.learned import learned_store from devplacepy.services.openai_gateway.shape import ShapeMethod class FakeFallbackShapeClient(FakeStrictClient_openai_gateway): rejected = () async def send(self, request, stream=False): body = request.json_body or {} self.calls.append((request.url, dict(body))) if body.get("model") == "shape-primary-target": return FakeResp_openai_gateway(status=503, payload={"error": {"message": "down"}}) return FakeResp_openai_gateway( payload={ "id": "x", "model": body.get("model"), "choices": [{"message": {"content": "ok"}}], } ) monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeFallbackShapeClient) routing.model_store.set( routing.ModelRouteIn(source_model="shape-backup", target_model="shape-backup-target") ) routing.model_store.set( routing.ModelRouteIn( source_model="shape-primary", target_model="shape-primary-target", fallback_model="shape-backup", ) ) learned_store.remember("default", "shape-backup-target", ShapeMethod("none")) try: svc, cfg = _shape_cfg_openai_gateway() rt = svc.runtime() response = run_async( rt.handle_chat( {"model": "shape-primary", "messages": [{"role": "user", "content": "hi"}]}, cfg, ("guest", "shape_fallback"), "test", "default", ) ) assert response.status_code == 200 assert "thinking" in rt._client.calls[0][1] assert rt._client.calls[-1][1]["model"] == "shape-backup-target" assert "thinking" not in rt._client.calls[-1][1] finally: learned_store.forget("default", "shape-backup-target") routing.model_store.remove("shape-primary") routing.model_store.remove("shape-backup") def _target_send_openai_gateway(rt, cfg, client, url, model): import asyncio async def go(): return await rt._send( client, asyncio.Semaphore(4), "POST", url, {}, {**cfg, "gateway_max_retries": 0}, lambda message: None, json_body={"model": model}, ) return run_async(go()) def _target_runtime_openai_gateway(): svc = GatewayService() return svc.runtime(), svc.effective_config() def test_trip_opens_below_threshold_and_ignores_threshold(): from devplacepy.services.openai_gateway.reliability import CircuitBreaker breaker = CircuitBreaker(5, 30) breaker.trip() assert breaker.is_open is True assert breaker.allow() is False def test_a_failed_target_is_skipped_without_an_upstream_call(local_db): rt, cfg = _target_runtime_openai_gateway() client = FakeAlwaysFailClient_openai_gateway() url = "https://target-a.example/v1/chat/completions" first = _target_send_openai_gateway(rt, cfg, client, url, "m1") assert first[0].status_code == 500 second = _target_send_openai_gateway(rt, cfg, client, url, "m1") assert second[2]["circuit_open"] is True assert len(client.calls) == 1 def test_one_bad_request_does_not_block_another_target(local_db): rt, cfg = _target_runtime_openai_gateway() client = FakeAlwaysFailClient_openai_gateway() url = "https://target-b.example/v1/chat/completions" _target_send_openai_gateway(rt, cfg, client, url, "model-b1") other = _target_send_openai_gateway(rt, cfg, client, url, "model-b2") assert other[2]["circuit_open"] is False assert len(client.calls) == 2 def test_target_breaker_kill_switch_keeps_global_behaviour(local_db): rt, cfg = _target_runtime_openai_gateway() cfg = {**cfg, "gateway_target_circuit_enabled": False} client = FakeAlwaysFailClient_openai_gateway() url = "https://target-d.example/v1/chat/completions" _target_send_openai_gateway(rt, cfg, client, url, "m1") again = _target_send_openai_gateway(rt, cfg, client, url, "m1") assert again[2]["circuit_open"] is False assert len(client.calls) == 2 def test_shape_fixable_400_does_not_trip_the_target(local_db): rt, cfg = _target_runtime_openai_gateway() resp = FakeResp_openai_gateway( status=400, payload={"issues": [{"code": "unrecognized_keys", "keys": ["thinking"]}]}, ) assert gwmod._trip_worthy(resp) is False plain = FakeResp_openai_gateway(status=400, payload={"error": "bad schema"}) assert gwmod._trip_worthy(plain) is True def test_embedded_200_error_trips_the_target(local_db): rt, cfg = _target_runtime_openai_gateway() resp = FakeResp_openai_gateway(status=200, payload={"error": {"message": "provider exploded"}}) url = "https://target-e.example/v1/chat/completions" assert rt._embedded_error(resp, False, url, "m1", cfg) is not None assert rt._target_breaker("m1", cfg).is_open is True assert rt.open_targets()[0]["model"] == "m1" def test_transition_notifies_each_admin_once_with_detail_and_no_key(local_db, monkeypatch): sent = [] monkeypatch.setattr(gwmod.background, "submit", lambda fn, *a, **k: sent.append((fn, a))) rt, cfg = _target_runtime_openai_gateway() url = "https://target-f.example/v1/chat/completions" rt._trip_target("m1", url, cfg, "upstream_error", 500, "Bearer sk-secret123 failed", 1) rt._trip_target("m1", url, cfg, "upstream_error", 500, "again", 1) assert len(sent) == 1 detail = sent[0][1][1] assert detail["category"] == "upstream_error" assert detail["status"] == 500 assert "secret123" not in detail["upstream"] assert detail["provider"] == "target-f.example" def test_breaker_is_per_model_across_upstreams(local_db): rt, cfg = _target_runtime_openai_gateway() client = FakeAlwaysFailClient_openai_gateway() _target_send_openai_gateway(rt, cfg, client, "https://target-g1.example/v1/chat/completions", "shared-model") second = _target_send_openai_gateway(rt, cfg, client, "https://target-g2.example/v1/chat/completions", "shared-model") assert second[2]["circuit_open"] is True assert len(client.calls) == 1 def test_a_request_without_a_model_never_uses_the_target_breaker(local_db): rt, cfg = _target_runtime_openai_gateway() client = FakeAlwaysFailClient_openai_gateway() url = "https://target-h.example/v1/other" _target_send_openai_gateway(rt, cfg, client, url, "") again = _target_send_openai_gateway(rt, cfg, client, url, "") assert again[2]["circuit_open"] is False assert rt._target_breakers == {} class FakeNotifyClient_openai_gateway: def __init__(self, *a, **k): self.calls = [] def build_request(self, method, url, headers=None, json=None, content=None): return FakeRequest(method, url, json) async def send(self, request, stream=False): self.calls.append((request.url, request.json_body)) body = request.json_body or {} if body.get("model") == "nb-primary-target": return FakeResp_openai_gateway( status=500, payload={"error": "boom sk-secret-xyz"} ) return FakeResp_openai_gateway( payload={ "id": "x", "model": body.get("model"), "choices": [{"message": {"content": "notify ok"}}], } ) async def aclose(self): pass class FakeNotifyChainClient_openai_gateway: def __init__(self, *a, **k): self.calls = [] def build_request(self, method, url, headers=None, json=None, content=None): return FakeRequest(method, url, json) async def send(self, request, stream=False): self.calls.append((request.url, request.json_body)) body = request.json_body or {} if body.get("model") in ("n3-a-target", "n3-b-target"): return FakeResp_openai_gateway(status=500, payload={"error": "boom"}) return FakeResp_openai_gateway( payload={ "id": "x", "model": body.get("model"), "choices": [{"message": {"content": "chain ok"}}], } ) async def aclose(self): pass def _notify_chat_openai_gateway(monkeypatch, client_cls, routes, source, owner_id): from devplacepy.services.openai_gateway import routing monkeypatch.setattr(gwmod.httpx, "AsyncClient", client_cls) for payload in routes: routing.model_store.set(routing.ModelRouteIn(**payload)) sent = [] def _capture_notify_openai_gateway(fn, *a, **k): if fn is gwmod._notify_gateway_status: sent.append((fn, a)) monkeypatch.setattr(gwmod.background, "submit", _capture_notify_openai_gateway) try: svc = GatewayService() cfg = svc.effective_config() cfg["gateway_vision_enabled"] = False cfg["gateway_max_retries"] = 0 rt = svc.runtime() response = run_async( rt.handle_chat( {"model": source, "messages": [{"role": "user", "content": "hi"}]}, cfg, ("guest", owner_id), "test", "default", ) ) return response, sent finally: for payload in routes: routing.model_store.remove(payload["source_model"]) def _rendered_messages_openai_gateway(monkeypatch, details): messages = [] class FakeUsersTable_openai_gateway: def find(self, **kwargs): if kwargs.get("role") == "Admin": return [{"uid": "admin-1"}] return [] monkeypatch.setattr( "devplacepy.database.get_table", lambda name: FakeUsersTable_openai_gateway(), ) monkeypatch.setattr( "devplacepy.utils.create_notification", lambda uid, kind, message, *a, **k: messages.append(message), ) for detail in details: gwmod._notify_gateway_status(True, detail) return messages def test_trip_notification_carries_local_config_redirect_and_outcome(local_db, monkeypatch): routes = [ {"source_model": "nb-backup", "target_model": "nb-backup-target"}, { "source_model": "nb-primary", "target_model": "nb-primary-target", "fallback_model": "nb-backup", }, ] response, sent = _notify_chat_openai_gateway( monkeypatch, FakeNotifyClient_openai_gateway, routes, "nb-primary", "notify_one" ) assert response.status_code == 200 assert len(sent) == 1 assert sent[0][0] is gwmod._notify_gateway_status detail = sent[0][1][1] assert detail["route"] == "nb-primary" assert detail["model"] == "nb-primary-target" assert detail["requested"] == "nb-primary" assert "nb-primary" in detail["chain"] assert "nb-backup" in detail["chain"] assert detail["redirect"] == "nb-backup" assert detail["outcome_ok"] is True assert detail["served_route"] == "nb-backup" assert "nb-backup" in detail["outcome"] messages = _rendered_messages_openai_gateway(monkeypatch, [detail]) assert len(messages) == 1 assert "tripped its circuit breaker" in messages[0] assert "Local config: 'nb-primary'" in messages[0] assert "Full route:" in messages[0] assert "nb-primary" in messages[0].split("Full route:")[1] assert "nb-backup" in messages[0].split("Full route:")[1] assert "Redirect: will redirect to fallback 'nb-backup'" in messages[0] assert "Outcome: fallback 'nb-backup' answered HTTP 200 ok" in messages[0] assert "Error class: upstream_error" in messages[0] assert "HTTP status: 500" in messages[0] assert "sk-secret-xyz" not in messages[0] assert "[redacted]" in messages[0] def test_trip_notifications_cover_every_hop_of_a_recursive_chain(local_db, monkeypatch): routes = [ {"source_model": "n3-c", "target_model": "n3-c-target"}, { "source_model": "n3-b", "target_model": "n3-b-target", "fallback_model": "n3-c", }, { "source_model": "n3-a", "target_model": "n3-a-target", "fallback_model": "n3-b", }, ] response, sent = _notify_chat_openai_gateway( monkeypatch, FakeNotifyChainClient_openai_gateway, routes, "n3-a", "notify_chain" ) assert response.status_code == 200 assert len(sent) == 2 first = sent[0][1][1] second = sent[1][1][1] assert first["route"] == "n3-a" assert first["redirect"] == "n3-b" assert second["route"] == "n3-b" assert second["redirect"] == "n3-c" for detail in (first, second): assert detail["requested"] == "n3-a" assert "n3-a" in detail["chain"] assert "n3-b" in detail["chain"] assert "n3-c" in detail["chain"] assert detail["outcome_ok"] is True assert detail["outcome"] == "fallback 'n3-c' answered HTTP 200 ok" assert detail["served_route"] == "n3-c" messages = _rendered_messages_openai_gateway(monkeypatch, [first, second]) assert "Redirect: will redirect to fallback 'n3-b'" in messages[0] assert "Redirect: will redirect to fallback 'n3-c'" in messages[1] assert "Outcome: fallback 'n3-c' answered HTTP 200 ok" in messages[0] assert "Outcome: fallback 'n3-c' answered HTTP 200 ok" in messages[1] def test_trip_notification_reports_failed_outcome_when_every_fallback_fails( local_db, monkeypatch ): routes = [ {"source_model": "n4-b", "target_model": "n4-b-target"}, { "source_model": "n4-a", "target_model": "n4-a-target", "fallback_model": "n4-b", }, ] response, sent = _notify_chat_openai_gateway( monkeypatch, FakeAlwaysFailClient_openai_gateway, routes, "n4-a", "notify_fail" ) assert response.status_code == 500 assert len(sent) == 2 first = sent[0][1][1] second = sent[1][1][1] assert first["route"] == "n4-a" assert first["redirect"] == "n4-b" assert second["route"] == "n4-b" assert second["redirect"] == "" for detail in (first, second): assert detail["outcome_ok"] is False assert "all 2 route(s) failed" in detail["outcome"] assert "HTTP 500" in detail["outcome"] messages = _rendered_messages_openai_gateway(monkeypatch, [first, second]) assert "Redirect: will redirect to fallback 'n4-b'" in messages[0] assert "Redirect: none, end of the fallback line" in messages[1] assert "Outcome: all 2 route(s) failed" in messages[1] def test_trip_notification_redirects_unrouted_default_to_first_fallback( local_db, monkeypatch ): rt, cfg = _target_runtime_openai_gateway() sent = [] monkeypatch.setattr( gwmod.background, "submit", lambda fn, *a, **k: sent.append((fn, a)) ) detail = gwmod._trip_detail( "default-target", "https://default.example/v1/chat/completions", 30, "upstream_error", 500, "boom", 0, route="", ) rt._notify_trips( [{"kind": "target", "detail": detail}], "chat", "molodetz", "weird-model", [{"route": ""}, {"route": "m-pro"}], True, "fallback 'm-pro' answered HTTP 200 ok", "m-pro", ) assert len(sent) == 1 enriched = sent[0][1][1] assert enriched["redirect"] == "m-pro" messages = _rendered_messages_openai_gateway(monkeypatch, [enriched]) assert "Local config: default (no routed model)" in messages[0] assert "Redirect: will redirect to fallback 'm-pro'" in messages[0] assert "Outcome: fallback 'm-pro' answered HTTP 200 ok" in messages[0] def test_vision_trip_notifies_with_local_config_and_no_fallback(local_db, monkeypatch): import httpx from devplacepy.services.openai_gateway.reliability import CircuitBreaker from devplacepy.services.openai_gateway.vision import VisionAugmenter, VisionCache class FailVisionClient_openai_gateway: async def post(self, url, headers=None, json=None): raise httpx.ConnectError("vision down") breaker = CircuitBreaker(1, 60) sink = [] augmenter = VisionAugmenter( "https://vision.example/v1/chat/completions", "vis-model", "k", VisionCache(0), breaker=breaker, max_retries=0, trip_sink=sink, ) image = {"type": "image_url", "image_url": {"url": "http://x/y.png"}} first = run_async(augmenter._describe_one(FailVisionClient_openai_gateway(), image)) assert first.startswith("[vision call failed") assert breaker.is_open is True assert len(sink) == 1 assert sink[0]["kind"] == "vision" assert sink[0]["detail"]["route"] == "vision:vis-model" second = run_async(augmenter._describe_one(FailVisionClient_openai_gateway(), image)) assert second.startswith("[vision unavailable") assert len(sink) == 1 rt, cfg = _target_runtime_openai_gateway() sent = [] monkeypatch.setattr( gwmod.background, "submit", lambda fn, *a, **k: sent.append((fn, a)) ) rt._notify_trips( sink, "chat", "molodetz", "molodetz", [{"route": "molodetz"}], True, "fallback 'molodetz' answered HTTP 200 ok", "molodetz", ) assert len(sent) == 1 messages = _rendered_messages_openai_gateway(monkeypatch, [sent[0][1][1]]) assert "Local config: 'vision:vis-model'" in messages[0] assert "Redirect: none, vision has no fallback model" in messages[0] assert "Outcome: fallback 'molodetz' answered HTTP 200 ok" in messages[0] def test_trip_notification_outcome_scrubs_secrets(local_db, monkeypatch): detail = gwmod._trip_detail( "s1-target", "https://s1.example/v1/chat/completions", 30, "connection", "no response", "boom", 0, route="s1", ) detail["requested"] = "s1" detail["chain"] = "s1 (s1-target @ default)" detail["redirect"] = "" secret = "sk" + "-abc123" detail["outcome"] = "connection failed (Bearer " + secret + ")" messages = _rendered_messages_openai_gateway(monkeypatch, [detail]) assert secret not in messages[0] assert "[redacted]" in messages[0] def test_passthrough_global_trip_carries_local_config(local_db, monkeypatch): monkeypatch.setattr( gwmod.httpx, "AsyncClient", FakeAlwaysFailClient_openai_gateway ) svc = GatewayService() cfg = svc.effective_config() cfg["gateway_circuit_threshold"] = 1 cfg["gateway_max_retries"] = 0 rt = svc.runtime() sent = [] monkeypatch.setattr( gwmod.background, "submit", lambda fn, *a, **k: sent.append((fn, a)) if fn is gwmod._notify_gateway_status else None, ) resp = run_async( rt.handle_passthrough( "POST", "responses", "application/json", b"{}", cfg, ("guest", "ptrip_local"), "test", "default", ) ) assert resp.status_code == 500 assert len(sent) == 1 detail = sent[0][1][1] assert detail.get("global") is True assert detail["route"] == cfg["gateway_model"] assert "no fallback line" in detail["chain"] messages = _rendered_messages_openai_gateway(monkeypatch, [detail]) assert f"Local config: '{cfg['gateway_model']}'" in messages[0] assert "passthrough POST responses (no fallback line)" in messages[0] assert "Outcome: all 1 route(s) failed" in messages[0]