Gateway: auto-resolve request shape per provider and model
An upstream that rejects a key the gateway adds (e.g. thinking) now gets retried with other shapes on a parsable 400/422; the working method is stored per (provider, target model) and applied first-try from then on. Fallback attempts are re-shaped per attempt. Admin list/reset, Devii tools, audit event and ledger resolved_method included. Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -3441,3 +3441,240 @@ def test_native_route_keeps_forwarding_tools(local_db, monkeypatch):
|
||||
assert "<tools>" not in sent["instructions"]
|
||||
finally:
|
||||
_teardown_responses_route_openai_gateway()
|
||||
|
||||
|
||||
VAIDS_REJECTION_openai_gateway = {
|
||||
"details": {"_errors": ["Unrecognized key(s) in object: 'thinking'"]},
|
||||
"error": "Invalid request parameters",
|
||||
"issues": [
|
||||
{
|
||||
"code": "unrecognized_keys",
|
||||
"keys": ["thinking"],
|
||||
"path": [],
|
||||
"message": "Unrecognized key(s) in object: 'thinking'",
|
||||
}
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
def _rejection_openai_gateway(key):
|
||||
return {
|
||||
"details": {"_errors": [f"Unrecognized key(s) in object: '{key}'"]},
|
||||
"error": "Invalid request parameters",
|
||||
"issues": [{"code": "unrecognized_keys", "keys": [key], "path": []}],
|
||||
}
|
||||
|
||||
|
||||
class FakeStrictClient_openai_gateway:
|
||||
rejected = ("thinking",)
|
||||
|
||||
def __init__(self, *a, **k):
|
||||
self.calls = []
|
||||
|
||||
def build_request(self, method, url, headers=None, json=None, content=None):
|
||||
return FakeRequest(method, url, json)
|
||||
|
||||
async def send(self, request, stream=False):
|
||||
body = request.json_body or {}
|
||||
self.calls.append((request.url, dict(body)))
|
||||
for key in self.rejected:
|
||||
if key in body:
|
||||
return FakeResp_openai_gateway(
|
||||
status=400, payload=_rejection_openai_gateway(key)
|
||||
)
|
||||
return FakeResp_openai_gateway(
|
||||
payload={
|
||||
"id": "x",
|
||||
"model": body.get("model"),
|
||||
"choices": [{"message": {"content": "hi there"}}],
|
||||
}
|
||||
)
|
||||
|
||||
async def aclose(self):
|
||||
pass
|
||||
|
||||
|
||||
class FakeStubbornClient_openai_gateway(FakeStrictClient_openai_gateway):
|
||||
async def send(self, request, stream=False):
|
||||
self.calls.append((request.url, dict(request.json_body or {})))
|
||||
return FakeResp_openai_gateway(
|
||||
status=400, payload=_rejection_openai_gateway("thinking")
|
||||
)
|
||||
|
||||
|
||||
class FakeReasoningStrictClient_openai_gateway(FakeStrictClient_openai_gateway):
|
||||
rejected = ("reasoning",)
|
||||
|
||||
|
||||
def _shape_cfg_openai_gateway():
|
||||
svc = GatewayService()
|
||||
cfg = svc.effective_config()
|
||||
cfg["gateway_vision_enabled"] = False
|
||||
cfg["gateway_max_retries"] = 0
|
||||
return svc, cfg
|
||||
|
||||
|
||||
def _chat_openai_gateway(rt, cfg, owner, stream=False):
|
||||
body = {"messages": [{"role": "user", "content": "hi"}]}
|
||||
if stream:
|
||||
body["stream"] = True
|
||||
return run_async(rt.handle_chat(body, cfg, ("guest", owner), "test", "default"))
|
||||
|
||||
|
||||
def test_chat_auto_resolves_a_rejected_thinking_key_and_remembers_it(local_db, monkeypatch):
|
||||
from devplacepy.services.openai_gateway.learned import learned_store
|
||||
|
||||
monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeStrictClient_openai_gateway)
|
||||
svc, cfg = _shape_cfg_openai_gateway()
|
||||
model = cfg["gateway_model"]
|
||||
learned_store.forget("default", model)
|
||||
try:
|
||||
rt = svc.runtime()
|
||||
response = _chat_openai_gateway(rt, cfg, "shape_resolve_first")
|
||||
assert response.status_code == 200
|
||||
assert len(rt._client.calls) == 2
|
||||
assert "thinking" in rt._client.calls[0][1]
|
||||
assert "thinking" not in rt._client.calls[1][1]
|
||||
learned = learned_store.get("default", model)
|
||||
assert learned is not None and learned.thinking_dialect == "none"
|
||||
row = get_table("gateway_usage_ledger").find_one(owner_id="shape_resolve_first")
|
||||
assert row["resolved_method"] == "none"
|
||||
assert row["success"] == 1
|
||||
second = _chat_openai_gateway(rt, cfg, "shape_resolve_second")
|
||||
assert second.status_code == 200
|
||||
assert len(rt._client.calls) == 3
|
||||
assert "thinking" not in rt._client.calls[2][1]
|
||||
finally:
|
||||
learned_store.forget("default", model)
|
||||
|
||||
|
||||
def test_chat_auto_resolve_works_for_streams(local_db, monkeypatch):
|
||||
from devplacepy.services.openai_gateway.learned import learned_store
|
||||
|
||||
monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeStrictClient_openai_gateway)
|
||||
svc, cfg = _shape_cfg_openai_gateway()
|
||||
model = cfg["gateway_model"]
|
||||
learned_store.forget("default", model)
|
||||
try:
|
||||
rt = svc.runtime()
|
||||
response = _chat_openai_gateway(rt, cfg, "shape_resolve_stream", stream=True)
|
||||
|
||||
async def drain():
|
||||
out = []
|
||||
async for chunk in response.body_iterator:
|
||||
out.append(chunk if isinstance(chunk, str) else chunk.decode())
|
||||
return "".join(out)
|
||||
|
||||
assert "[DONE]" in run_async(drain())
|
||||
assert len(rt._client.calls) == 2
|
||||
assert learned_store.get("default", model) is not None
|
||||
finally:
|
||||
learned_store.forget("default", model)
|
||||
|
||||
|
||||
def test_chat_genuine_bad_request_is_not_auto_resolved(local_db, monkeypatch):
|
||||
from devplacepy.services.openai_gateway.learned import learned_store
|
||||
|
||||
monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeBadRequestClient_openai_gateway)
|
||||
svc, cfg = _shape_cfg_openai_gateway()
|
||||
model = cfg["gateway_model"]
|
||||
learned_store.forget("default", model)
|
||||
rt = svc.runtime()
|
||||
response = _chat_openai_gateway(rt, cfg, "shape_genuine_bad")
|
||||
assert response.status_code == 400
|
||||
assert len(rt._client.calls) == 1
|
||||
assert learned_store.get("default", model) is None
|
||||
|
||||
|
||||
def test_chat_exhausted_resolution_returns_the_original_error_and_stores_nothing(
|
||||
local_db, monkeypatch
|
||||
):
|
||||
from devplacepy.services.openai_gateway.learned import learned_store
|
||||
from devplacepy.services.openai_gateway.shape import MAX_RESOLVE_ATTEMPTS
|
||||
|
||||
monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeStubbornClient_openai_gateway)
|
||||
svc, cfg = _shape_cfg_openai_gateway()
|
||||
model = cfg["gateway_model"]
|
||||
learned_store.forget("default", model)
|
||||
rt = svc.runtime()
|
||||
response = _chat_openai_gateway(rt, cfg, "shape_exhausted")
|
||||
assert response.status_code == 400
|
||||
assert "thinking" in response.body.decode()
|
||||
assert len(rt._client.calls) <= 1 + MAX_RESOLVE_ATTEMPTS
|
||||
assert learned_store.get("default", model) is None
|
||||
|
||||
|
||||
def test_chat_rejected_learned_method_is_forgotten_and_ladder_restarts(local_db, monkeypatch):
|
||||
from devplacepy.services.openai_gateway.learned import learned_store
|
||||
from devplacepy.services.openai_gateway.shape import ShapeMethod
|
||||
|
||||
monkeypatch.setattr(
|
||||
gwmod.httpx, "AsyncClient", FakeReasoningStrictClient_openai_gateway
|
||||
)
|
||||
svc, cfg = _shape_cfg_openai_gateway()
|
||||
model = cfg["gateway_model"]
|
||||
learned_store.remember("default", model, ShapeMethod("openrouter"))
|
||||
try:
|
||||
rt = svc.runtime()
|
||||
response = _chat_openai_gateway(rt, cfg, "shape_heal")
|
||||
assert response.status_code == 200
|
||||
assert "reasoning" in rt._client.calls[0][1]
|
||||
assert "thinking" in rt._client.calls[1][1]
|
||||
assert learned_store.get("default", model) is None
|
||||
finally:
|
||||
learned_store.forget("default", model)
|
||||
|
||||
|
||||
def test_fallback_attempt_is_shaped_with_its_own_learned_method(local_db, monkeypatch):
|
||||
from devplacepy.services.openai_gateway import routing
|
||||
from devplacepy.services.openai_gateway.learned import learned_store
|
||||
from devplacepy.services.openai_gateway.shape import ShapeMethod
|
||||
|
||||
class FakeFallbackShapeClient(FakeStrictClient_openai_gateway):
|
||||
rejected = ()
|
||||
|
||||
async def send(self, request, stream=False):
|
||||
body = request.json_body or {}
|
||||
self.calls.append((request.url, dict(body)))
|
||||
if body.get("model") == "shape-primary-target":
|
||||
return FakeResp_openai_gateway(status=503, payload={"error": {"message": "down"}})
|
||||
return FakeResp_openai_gateway(
|
||||
payload={
|
||||
"id": "x",
|
||||
"model": body.get("model"),
|
||||
"choices": [{"message": {"content": "ok"}}],
|
||||
}
|
||||
)
|
||||
|
||||
monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeFallbackShapeClient)
|
||||
routing.model_store.set(
|
||||
routing.ModelRouteIn(source_model="shape-backup", target_model="shape-backup-target")
|
||||
)
|
||||
routing.model_store.set(
|
||||
routing.ModelRouteIn(
|
||||
source_model="shape-primary",
|
||||
target_model="shape-primary-target",
|
||||
fallback_model="shape-backup",
|
||||
)
|
||||
)
|
||||
learned_store.remember("default", "shape-backup-target", ShapeMethod("none"))
|
||||
try:
|
||||
svc, cfg = _shape_cfg_openai_gateway()
|
||||
rt = svc.runtime()
|
||||
response = run_async(
|
||||
rt.handle_chat(
|
||||
{"model": "shape-primary", "messages": [{"role": "user", "content": "hi"}]},
|
||||
cfg,
|
||||
("guest", "shape_fallback"),
|
||||
"test",
|
||||
"default",
|
||||
)
|
||||
)
|
||||
assert response.status_code == 200
|
||||
assert "thinking" in rt._client.calls[0][1]
|
||||
assert rt._client.calls[-1][1]["model"] == "shape-backup-target"
|
||||
assert "thinking" not in rt._client.calls[-1][1]
|
||||
finally:
|
||||
learned_store.forget("default", "shape-backup-target")
|
||||
routing.model_store.remove("shape-primary")
|
||||
routing.model_store.remove("shape-backup")
|
||||
|
||||
Reference in New Issue
Block a user