Gateway: auto-resolve request shape per provider and model

An upstream that rejects a key the gateway adds (e.g. thinking) now gets
retried with other shapes on a parsable 400/422; the working method is
stored per (provider, target model) and applied first-try from then on.
Fallback attempts are re-shaped per attempt. Admin list/reset, Devii
tools, audit event and ledger resolved_method included.

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
2026-11-13 11:34:26 +01:00
co-authored by Claude Sonnet 5.5
parent 3f6e83c753
commit 151476125f
22 changed files with 1597 additions and 129 deletions
@@ -3441,3 +3441,240 @@ def test_native_route_keeps_forwarding_tools(local_db, monkeypatch):
assert "<tools>" not in sent["instructions"]
finally:
_teardown_responses_route_openai_gateway()
VAIDS_REJECTION_openai_gateway = {
"details": {"_errors": ["Unrecognized key(s) in object: 'thinking'"]},
"error": "Invalid request parameters",
"issues": [
{
"code": "unrecognized_keys",
"keys": ["thinking"],
"path": [],
"message": "Unrecognized key(s) in object: 'thinking'",
}
],
}
def _rejection_openai_gateway(key):
return {
"details": {"_errors": [f"Unrecognized key(s) in object: '{key}'"]},
"error": "Invalid request parameters",
"issues": [{"code": "unrecognized_keys", "keys": [key], "path": []}],
}
class FakeStrictClient_openai_gateway:
rejected = ("thinking",)
def __init__(self, *a, **k):
self.calls = []
def build_request(self, method, url, headers=None, json=None, content=None):
return FakeRequest(method, url, json)
async def send(self, request, stream=False):
body = request.json_body or {}
self.calls.append((request.url, dict(body)))
for key in self.rejected:
if key in body:
return FakeResp_openai_gateway(
status=400, payload=_rejection_openai_gateway(key)
)
return FakeResp_openai_gateway(
payload={
"id": "x",
"model": body.get("model"),
"choices": [{"message": {"content": "hi there"}}],
}
)
async def aclose(self):
pass
class FakeStubbornClient_openai_gateway(FakeStrictClient_openai_gateway):
async def send(self, request, stream=False):
self.calls.append((request.url, dict(request.json_body or {})))
return FakeResp_openai_gateway(
status=400, payload=_rejection_openai_gateway("thinking")
)
class FakeReasoningStrictClient_openai_gateway(FakeStrictClient_openai_gateway):
rejected = ("reasoning",)
def _shape_cfg_openai_gateway():
svc = GatewayService()
cfg = svc.effective_config()
cfg["gateway_vision_enabled"] = False
cfg["gateway_max_retries"] = 0
return svc, cfg
def _chat_openai_gateway(rt, cfg, owner, stream=False):
body = {"messages": [{"role": "user", "content": "hi"}]}
if stream:
body["stream"] = True
return run_async(rt.handle_chat(body, cfg, ("guest", owner), "test", "default"))
def test_chat_auto_resolves_a_rejected_thinking_key_and_remembers_it(local_db, monkeypatch):
from devplacepy.services.openai_gateway.learned import learned_store
monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeStrictClient_openai_gateway)
svc, cfg = _shape_cfg_openai_gateway()
model = cfg["gateway_model"]
learned_store.forget("default", model)
try:
rt = svc.runtime()
response = _chat_openai_gateway(rt, cfg, "shape_resolve_first")
assert response.status_code == 200
assert len(rt._client.calls) == 2
assert "thinking" in rt._client.calls[0][1]
assert "thinking" not in rt._client.calls[1][1]
learned = learned_store.get("default", model)
assert learned is not None and learned.thinking_dialect == "none"
row = get_table("gateway_usage_ledger").find_one(owner_id="shape_resolve_first")
assert row["resolved_method"] == "none"
assert row["success"] == 1
second = _chat_openai_gateway(rt, cfg, "shape_resolve_second")
assert second.status_code == 200
assert len(rt._client.calls) == 3
assert "thinking" not in rt._client.calls[2][1]
finally:
learned_store.forget("default", model)
def test_chat_auto_resolve_works_for_streams(local_db, monkeypatch):
from devplacepy.services.openai_gateway.learned import learned_store
monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeStrictClient_openai_gateway)
svc, cfg = _shape_cfg_openai_gateway()
model = cfg["gateway_model"]
learned_store.forget("default", model)
try:
rt = svc.runtime()
response = _chat_openai_gateway(rt, cfg, "shape_resolve_stream", stream=True)
async def drain():
out = []
async for chunk in response.body_iterator:
out.append(chunk if isinstance(chunk, str) else chunk.decode())
return "".join(out)
assert "[DONE]" in run_async(drain())
assert len(rt._client.calls) == 2
assert learned_store.get("default", model) is not None
finally:
learned_store.forget("default", model)
def test_chat_genuine_bad_request_is_not_auto_resolved(local_db, monkeypatch):
from devplacepy.services.openai_gateway.learned import learned_store
monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeBadRequestClient_openai_gateway)
svc, cfg = _shape_cfg_openai_gateway()
model = cfg["gateway_model"]
learned_store.forget("default", model)
rt = svc.runtime()
response = _chat_openai_gateway(rt, cfg, "shape_genuine_bad")
assert response.status_code == 400
assert len(rt._client.calls) == 1
assert learned_store.get("default", model) is None
def test_chat_exhausted_resolution_returns_the_original_error_and_stores_nothing(
local_db, monkeypatch
):
from devplacepy.services.openai_gateway.learned import learned_store
from devplacepy.services.openai_gateway.shape import MAX_RESOLVE_ATTEMPTS
monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeStubbornClient_openai_gateway)
svc, cfg = _shape_cfg_openai_gateway()
model = cfg["gateway_model"]
learned_store.forget("default", model)
rt = svc.runtime()
response = _chat_openai_gateway(rt, cfg, "shape_exhausted")
assert response.status_code == 400
assert "thinking" in response.body.decode()
assert len(rt._client.calls) <= 1 + MAX_RESOLVE_ATTEMPTS
assert learned_store.get("default", model) is None
def test_chat_rejected_learned_method_is_forgotten_and_ladder_restarts(local_db, monkeypatch):
from devplacepy.services.openai_gateway.learned import learned_store
from devplacepy.services.openai_gateway.shape import ShapeMethod
monkeypatch.setattr(
gwmod.httpx, "AsyncClient", FakeReasoningStrictClient_openai_gateway
)
svc, cfg = _shape_cfg_openai_gateway()
model = cfg["gateway_model"]
learned_store.remember("default", model, ShapeMethod("openrouter"))
try:
rt = svc.runtime()
response = _chat_openai_gateway(rt, cfg, "shape_heal")
assert response.status_code == 200
assert "reasoning" in rt._client.calls[0][1]
assert "thinking" in rt._client.calls[1][1]
assert learned_store.get("default", model) is None
finally:
learned_store.forget("default", model)
def test_fallback_attempt_is_shaped_with_its_own_learned_method(local_db, monkeypatch):
from devplacepy.services.openai_gateway import routing
from devplacepy.services.openai_gateway.learned import learned_store
from devplacepy.services.openai_gateway.shape import ShapeMethod
class FakeFallbackShapeClient(FakeStrictClient_openai_gateway):
rejected = ()
async def send(self, request, stream=False):
body = request.json_body or {}
self.calls.append((request.url, dict(body)))
if body.get("model") == "shape-primary-target":
return FakeResp_openai_gateway(status=503, payload={"error": {"message": "down"}})
return FakeResp_openai_gateway(
payload={
"id": "x",
"model": body.get("model"),
"choices": [{"message": {"content": "ok"}}],
}
)
monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeFallbackShapeClient)
routing.model_store.set(
routing.ModelRouteIn(source_model="shape-backup", target_model="shape-backup-target")
)
routing.model_store.set(
routing.ModelRouteIn(
source_model="shape-primary",
target_model="shape-primary-target",
fallback_model="shape-backup",
)
)
learned_store.remember("default", "shape-backup-target", ShapeMethod("none"))
try:
svc, cfg = _shape_cfg_openai_gateway()
rt = svc.runtime()
response = run_async(
rt.handle_chat(
{"model": "shape-primary", "messages": [{"role": "user", "content": "hi"}]},
cfg,
("guest", "shape_fallback"),
"test",
"default",
)
)
assert response.status_code == 200
assert "thinking" in rt._client.calls[0][1]
assert rt._client.calls[-1][1]["model"] == "shape-backup-target"
assert "thinking" not in rt._client.calls[-1][1]
finally:
learned_store.forget("default", "shape-backup-target")
routing.model_store.remove("shape-primary")
routing.model_store.remove("shape-backup")