Gateway: auto-resolve request shape per provider and model

An upstream that rejects a key the gateway adds (e.g. thinking) now gets
retried with other shapes on a parsable 400/422; the working method is
stored per (provider, target model) and applied first-try from then on.
Fallback attempts are re-shaped per attempt. Admin list/reset, Devii
tools, audit event and ledger resolved_method included.

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
2026-11-13 11:34:26 +01:00
co-authored by Claude Sonnet 5.5
parent 3f6e83c753
commit 151476125f
22 changed files with 1597 additions and 129 deletions
@@ -3441,3 +3441,240 @@ def test_native_route_keeps_forwarding_tools(local_db, monkeypatch):
assert "<tools>" not in sent["instructions"]
finally:
_teardown_responses_route_openai_gateway()
VAIDS_REJECTION_openai_gateway = {
"details": {"_errors": ["Unrecognized key(s) in object: 'thinking'"]},
"error": "Invalid request parameters",
"issues": [
{
"code": "unrecognized_keys",
"keys": ["thinking"],
"path": [],
"message": "Unrecognized key(s) in object: 'thinking'",
}
],
}
def _rejection_openai_gateway(key):
return {
"details": {"_errors": [f"Unrecognized key(s) in object: '{key}'"]},
"error": "Invalid request parameters",
"issues": [{"code": "unrecognized_keys", "keys": [key], "path": []}],
}
class FakeStrictClient_openai_gateway:
rejected = ("thinking",)
def __init__(self, *a, **k):
self.calls = []
def build_request(self, method, url, headers=None, json=None, content=None):
return FakeRequest(method, url, json)
async def send(self, request, stream=False):
body = request.json_body or {}
self.calls.append((request.url, dict(body)))
for key in self.rejected:
if key in body:
return FakeResp_openai_gateway(
status=400, payload=_rejection_openai_gateway(key)
)
return FakeResp_openai_gateway(
payload={
"id": "x",
"model": body.get("model"),
"choices": [{"message": {"content": "hi there"}}],
}
)
async def aclose(self):
pass
class FakeStubbornClient_openai_gateway(FakeStrictClient_openai_gateway):
async def send(self, request, stream=False):
self.calls.append((request.url, dict(request.json_body or {})))
return FakeResp_openai_gateway(
status=400, payload=_rejection_openai_gateway("thinking")
)
class FakeReasoningStrictClient_openai_gateway(FakeStrictClient_openai_gateway):
rejected = ("reasoning",)
def _shape_cfg_openai_gateway():
svc = GatewayService()
cfg = svc.effective_config()
cfg["gateway_vision_enabled"] = False
cfg["gateway_max_retries"] = 0
return svc, cfg
def _chat_openai_gateway(rt, cfg, owner, stream=False):
body = {"messages": [{"role": "user", "content": "hi"}]}
if stream:
body["stream"] = True
return run_async(rt.handle_chat(body, cfg, ("guest", owner), "test", "default"))
def test_chat_auto_resolves_a_rejected_thinking_key_and_remembers_it(local_db, monkeypatch):
from devplacepy.services.openai_gateway.learned import learned_store
monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeStrictClient_openai_gateway)
svc, cfg = _shape_cfg_openai_gateway()
model = cfg["gateway_model"]
learned_store.forget("default", model)
try:
rt = svc.runtime()
response = _chat_openai_gateway(rt, cfg, "shape_resolve_first")
assert response.status_code == 200
assert len(rt._client.calls) == 2
assert "thinking" in rt._client.calls[0][1]
assert "thinking" not in rt._client.calls[1][1]
learned = learned_store.get("default", model)
assert learned is not None and learned.thinking_dialect == "none"
row = get_table("gateway_usage_ledger").find_one(owner_id="shape_resolve_first")
assert row["resolved_method"] == "none"
assert row["success"] == 1
second = _chat_openai_gateway(rt, cfg, "shape_resolve_second")
assert second.status_code == 200
assert len(rt._client.calls) == 3
assert "thinking" not in rt._client.calls[2][1]
finally:
learned_store.forget("default", model)
def test_chat_auto_resolve_works_for_streams(local_db, monkeypatch):
from devplacepy.services.openai_gateway.learned import learned_store
monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeStrictClient_openai_gateway)
svc, cfg = _shape_cfg_openai_gateway()
model = cfg["gateway_model"]
learned_store.forget("default", model)
try:
rt = svc.runtime()
response = _chat_openai_gateway(rt, cfg, "shape_resolve_stream", stream=True)
async def drain():
out = []
async for chunk in response.body_iterator:
out.append(chunk if isinstance(chunk, str) else chunk.decode())
return "".join(out)
assert "[DONE]" in run_async(drain())
assert len(rt._client.calls) == 2
assert learned_store.get("default", model) is not None
finally:
learned_store.forget("default", model)
def test_chat_genuine_bad_request_is_not_auto_resolved(local_db, monkeypatch):
from devplacepy.services.openai_gateway.learned import learned_store
monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeBadRequestClient_openai_gateway)
svc, cfg = _shape_cfg_openai_gateway()
model = cfg["gateway_model"]
learned_store.forget("default", model)
rt = svc.runtime()
response = _chat_openai_gateway(rt, cfg, "shape_genuine_bad")
assert response.status_code == 400
assert len(rt._client.calls) == 1
assert learned_store.get("default", model) is None
def test_chat_exhausted_resolution_returns_the_original_error_and_stores_nothing(
local_db, monkeypatch
):
from devplacepy.services.openai_gateway.learned import learned_store
from devplacepy.services.openai_gateway.shape import MAX_RESOLVE_ATTEMPTS
monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeStubbornClient_openai_gateway)
svc, cfg = _shape_cfg_openai_gateway()
model = cfg["gateway_model"]
learned_store.forget("default", model)
rt = svc.runtime()
response = _chat_openai_gateway(rt, cfg, "shape_exhausted")
assert response.status_code == 400
assert "thinking" in response.body.decode()
assert len(rt._client.calls) <= 1 + MAX_RESOLVE_ATTEMPTS
assert learned_store.get("default", model) is None
def test_chat_rejected_learned_method_is_forgotten_and_ladder_restarts(local_db, monkeypatch):
from devplacepy.services.openai_gateway.learned import learned_store
from devplacepy.services.openai_gateway.shape import ShapeMethod
monkeypatch.setattr(
gwmod.httpx, "AsyncClient", FakeReasoningStrictClient_openai_gateway
)
svc, cfg = _shape_cfg_openai_gateway()
model = cfg["gateway_model"]
learned_store.remember("default", model, ShapeMethod("openrouter"))
try:
rt = svc.runtime()
response = _chat_openai_gateway(rt, cfg, "shape_heal")
assert response.status_code == 200
assert "reasoning" in rt._client.calls[0][1]
assert "thinking" in rt._client.calls[1][1]
assert learned_store.get("default", model) is None
finally:
learned_store.forget("default", model)
def test_fallback_attempt_is_shaped_with_its_own_learned_method(local_db, monkeypatch):
from devplacepy.services.openai_gateway import routing
from devplacepy.services.openai_gateway.learned import learned_store
from devplacepy.services.openai_gateway.shape import ShapeMethod
class FakeFallbackShapeClient(FakeStrictClient_openai_gateway):
rejected = ()
async def send(self, request, stream=False):
body = request.json_body or {}
self.calls.append((request.url, dict(body)))
if body.get("model") == "shape-primary-target":
return FakeResp_openai_gateway(status=503, payload={"error": {"message": "down"}})
return FakeResp_openai_gateway(
payload={
"id": "x",
"model": body.get("model"),
"choices": [{"message": {"content": "ok"}}],
}
)
monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeFallbackShapeClient)
routing.model_store.set(
routing.ModelRouteIn(source_model="shape-backup", target_model="shape-backup-target")
)
routing.model_store.set(
routing.ModelRouteIn(
source_model="shape-primary",
target_model="shape-primary-target",
fallback_model="shape-backup",
)
)
learned_store.remember("default", "shape-backup-target", ShapeMethod("none"))
try:
svc, cfg = _shape_cfg_openai_gateway()
rt = svc.runtime()
response = run_async(
rt.handle_chat(
{"model": "shape-primary", "messages": [{"role": "user", "content": "hi"}]},
cfg,
("guest", "shape_fallback"),
"test",
"default",
)
)
assert response.status_code == 200
assert "thinking" in rt._client.calls[0][1]
assert rt._client.calls[-1][1]["model"] == "shape-backup-target"
assert "thinking" not in rt._client.calls[-1][1]
finally:
learned_store.forget("default", "shape-backup-target")
routing.model_store.remove("shape-primary")
routing.model_store.remove("shape-backup")
@@ -0,0 +1,90 @@
# retoor <retoor@molodetz.nl>
from devplacepy.services.openai_gateway import learned
from devplacepy.services.openai_gateway.learned import learned_store
from devplacepy.services.openai_gateway.shape import NATIVE_METHOD, ShapeMethod
def test_remember_get_list_and_forget(local_db):
learned_store.forget("lt-prov", "lt-model")
method = ShapeMethod("none", ("stream_options",))
learned_store.remember("LT-Prov", "lt-model", method)
try:
assert learned_store.get("lt-prov", "lt-model") == method
rows = [r for r in learned_store.list() if r["target_model"] == "lt-model"]
assert rows[0]["provider"] == "lt-prov"
assert rows[0]["drop_keys"] == ["stream_options"]
assert learned_store.forget("lt-prov", "lt-model") is True
assert learned_store.get("lt-prov", "lt-model") is None
assert learned_store.forget("lt-prov", "lt-model") is False
finally:
learned_store.forget("lt-prov", "lt-model")
def test_blank_provider_maps_to_default_and_native_is_never_stored(local_db):
learned_store.forget("default", "lt-default-model")
learned_store.remember("", "lt-default-model", NATIVE_METHOD)
assert learned_store.get("default", "lt-default-model") is None
learned_store.remember("", "lt-default-model", ShapeMethod("none"))
try:
assert learned_store.get("default", "lt-default-model") == ShapeMethod("none")
finally:
learned_store.forget("default", "lt-default-model")
def test_remember_upserts_on_the_unique_route(local_db):
learned_store.forget("lt-up", "lt-m")
try:
learned_store.remember("lt-up", "lt-m", ShapeMethod("none"))
learned_store.remember("lt-up", "lt-m", ShapeMethod("openrouter"))
assert learned_store.get("lt-up", "lt-m") == ShapeMethod("openrouter")
assert len([r for r in learned_store.list() if r["provider"] == "lt-up"]) == 1
finally:
learned_store.forget("lt-up", "lt-m")
def test_forget_provider_clears_only_that_provider(local_db):
learned_store.remember("lt-fp", "a", ShapeMethod("none"))
learned_store.remember("lt-fp", "b", ShapeMethod("none"))
learned_store.remember("lt-other", "a", ShapeMethod("none"))
try:
assert learned_store.forget_provider("lt-fp") == 2
assert learned_store.get("lt-fp", "a") is None
assert learned_store.get("lt-other", "a") is not None
finally:
learned_store.forget_provider("lt-fp")
learned_store.forget_provider("lt-other")
def test_row_cap_replaces_the_oldest(local_db, monkeypatch):
monkeypatch.setattr(learned, "MAX_LEARNED_ROWS", 2)
keys = [("lt-cap", f"m{i}") for i in range(3)]
try:
for provider, model in keys:
learned_store.remember(provider, model, ShapeMethod("none"))
present = [k for k in keys if learned_store.get(*k) is not None]
assert len(present) <= 2
assert learned_store.get(*keys[2]) is not None
finally:
learned_store.forget_provider("lt-cap")
def test_provider_endpoint_edit_forgets_learned_methods(local_db):
from devplacepy.services.openai_gateway import routing
routing.provider_store.set(
routing.ProviderIn(name="lt-edit", base_url="https://a.example/v1/chat/completions")
)
learned_store.remember("lt-edit", "m", ShapeMethod("none"))
try:
routing.provider_store.set(
routing.ProviderIn(name="lt-edit", base_url="https://a.example/v1/chat/completions")
)
assert learned_store.get("lt-edit", "m") is not None
routing.provider_store.set(
routing.ProviderIn(name="lt-edit", base_url="https://b.example/v1/chat/completions")
)
assert learned_store.get("lt-edit", "m") is None
finally:
routing.provider_store.remove("lt-edit")
learned_store.forget_provider("lt-edit")
+174
View File
@@ -0,0 +1,174 @@
# retoor <retoor@molodetz.nl>
import json
from devplacepy.services.openai_gateway.shape import (
MAX_RESOLVE_ATTEMPTS,
NATIVE_METHOD,
RejectedKeys,
ShapeMethod,
ShapeResolver,
ShapeRetrier,
)
VAIDS_BODY = json.dumps(
{
"details": {"_errors": ["Unrecognized key(s) in object: 'thinking'"]},
"error": "Invalid request parameters",
"issues": [
{
"code": "unrecognized_keys",
"keys": ["thinking"],
"path": [],
"message": "Unrecognized key(s) in object: 'thinking'",
}
],
}
)
class MemoryStore_openai_gateway:
def __init__(self):
self.rows = {}
self.forgotten = []
def get(self, provider, model):
return self.rows.get((provider, model))
def remember(self, provider, model, method):
self.rows[(provider, model)] = method
def forget(self, provider, model):
self.forgotten.append((provider, model))
return self.rows.pop((provider, model), None) is not None
def test_parse_real_vaids_body():
assert RejectedKeys.parse(VAIDS_BODY) == ("thinking",)
def test_parse_openai_style_bodies():
assert RejectedKeys.parse(
'{"error":{"message":"Unrecognized request argument supplied: foo","param":null}}'
) == ("foo",)
assert RejectedKeys.parse(
'{"error":{"message":"Unknown parameter: \'stream_options\'.",'
'"param":"stream_options","code":"unknown_parameter"}}'
) == ("stream_options",)
def test_parse_serde_style_and_plain_text():
assert RejectedKeys.parse("unknown field `thinking`, expected one of `model`") == (
"thinking",
)
def test_parse_returns_empty_for_a_genuine_bad_request():
assert RejectedKeys.parse('{"error":{"message":"invalid request: bad param"}}') == ()
assert RejectedKeys.parse("") == ()
assert RejectedKeys.parse(None) == ()
def test_resolver_prefers_none_when_thinking_is_rejected():
resolver = ShapeResolver("deepseek", "chat_completions")
method = resolver.next_method({NATIVE_METHOD}, NATIVE_METHOD, ("thinking",))
assert method == ShapeMethod("none")
def test_resolver_skips_dialects_that_emit_the_rejected_key():
resolver = ShapeResolver("deepseek", "chat_completions")
tried = {NATIVE_METHOD, ShapeMethod("none")}
method = resolver.next_method(tried, ShapeMethod("none"), ("reasoning",))
assert method == ShapeMethod("deepseek")
def test_resolver_adds_a_droppable_key():
resolver = ShapeResolver("deepseek", "chat_completions")
method = resolver.next_method({NATIVE_METHOD}, NATIVE_METHOD, ("stream_options",))
assert method == ShapeMethod("native", ("stream_options",))
def test_resolver_refuses_unknown_and_protected_keys():
resolver = ShapeResolver("deepseek", "chat_completions")
for key in ("messages", "model", "tools", "something_else"):
assert resolver.next_method({NATIVE_METHOD}, NATIVE_METHOD, (key,)) is None
def test_resolver_responses_ladder_is_reasoning_only():
resolver = ShapeResolver("deepseek", "responses")
method = resolver.next_method({NATIVE_METHOD}, NATIVE_METHOD, ("reasoning",))
assert method == ShapeMethod("none")
assert resolver.next_method({NATIVE_METHOD}, NATIVE_METHOD, ("thinking",)) is None
def test_method_json_round_trip_and_validation():
method = ShapeMethod("none", ("stream_options",))
assert ShapeMethod.from_json(method.to_json()) == method
assert ShapeMethod.from_json('{"thinking_dialect":"bogus","drop_keys":[]}') is None
assert ShapeMethod.from_json('{"thinking_dialect":"none","drop_keys":["tools"]}') is None
assert ShapeMethod.from_json("not json") is None
def test_apply_none_clears_thinking_and_native_writes_the_dialect():
cfg = {"gateway_upstream_url": "https://strict.example/v1/chat/completions"}
payload = {"model": "m", "messages": [], "thinking": {"type": "enabled"}}
native = NATIVE_METHOD.apply(payload, cfg, "chat_completions", False, lambda m: None)
assert native["thinking"] == {"type": "enabled"}
none = ShapeMethod("none").apply(payload, cfg, "chat_completions", False, lambda m: None)
assert "thinking" not in none and "reasoning" not in none
assert payload["thinking"] == {"type": "enabled"}
def test_apply_drops_stream_options_when_learned():
cfg = {"gateway_upstream_url": "https://strict.example/v1/chat/completions"}
payload = {"model": "m", "messages": [], "stream": True}
kept = NATIVE_METHOD.apply(payload, cfg, "chat_completions", True, lambda m: None)
assert kept["stream_options"] == {"include_usage": True}
dropped = ShapeMethod("native", ("stream_options",)).apply(
payload, cfg, "chat_completions", True, lambda m: None
)
assert "stream_options" not in dropped
def test_strip_wire_removes_reasoning_for_responses_none():
body = {"model": "m", "reasoning": {"effort": "minimal"}, "input": []}
stripped = ShapeMethod("none").strip_wire(body, "responses")
assert "reasoning" not in stripped and body["reasoning"]
assert NATIVE_METHOD.strip_wire(body, "responses") is body
def test_retrier_success_remembers_and_native_success_does_not():
store = MemoryStore_openai_gateway()
retrier = ShapeRetrier.for_route(store, "p", "m", "chat_completions", "deepseek", lambda m: None)
retrier.succeeded()
assert store.rows == {}
method = retrier.next_after(400, VAIDS_BODY)
assert method == ShapeMethod("none")
retrier.succeeded()
assert store.rows[("p", "m")] == ShapeMethod("none")
def test_retrier_ignores_non_shape_statuses_and_unparsable_bodies():
store = MemoryStore_openai_gateway()
retrier = ShapeRetrier.for_route(store, "p", "m", "chat_completions", "deepseek", lambda m: None)
assert retrier.next_after(500, VAIDS_BODY) is None
assert retrier.next_after(400, '{"error":"nope"}') is None
def test_retrier_attempts_are_capped():
store = MemoryStore_openai_gateway()
retrier = ShapeRetrier.for_route(store, "p", "m", "chat_completions", "deepseek", lambda m: None)
retrier.attempts = MAX_RESOLVE_ATTEMPTS
assert retrier.next_after(400, VAIDS_BODY) is None
def test_retrier_forgets_a_rejected_learned_method_once():
store = MemoryStore_openai_gateway()
store.rows[("p", "m")] = ShapeMethod("openrouter")
retrier = ShapeRetrier.for_route(store, "p", "m", "chat_completions", "deepseek", lambda m: None)
assert retrier.method == ShapeMethod("openrouter")
method = retrier.next_after(
400, '{"issues":[{"code":"unrecognized_keys","keys":["reasoning"]}]}'
)
assert method == NATIVE_METHOD
assert store.forgotten == [("p", "m")]