Gateway: auto-resolve request shape per provider and model

An upstream that rejects a key the gateway adds (e.g. thinking) now gets
retried with other shapes on a parsable 400/422; the working method is
stored per (provider, target model) and applied first-try from then on.
Fallback attempts are re-shaped per attempt. Admin list/reset, Devii
tools, audit event and ledger resolved_method included.

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
2026-11-13 11:34:26 +01:00
co-authored by Claude Sonnet 5.5
parent 3f6e83c753
commit 151476125f
22 changed files with 1597 additions and 129 deletions
+77
View File
@@ -0,0 +1,77 @@
# retoor <retoor@molodetz.nl>
import time
import requests
from devplacepy.database import refresh_snapshot
from devplacepy.services.openai_gateway.learned import learned_store
from devplacepy.services.openai_gateway.shape import ShapeMethod
from tests.api.admin.gateway.index import (
JSON_gateway,
admin_session,
member_key,
_unique_gateway,
)
from tests.conftest import BASE_URL
LIST_URL = f"{BASE_URL}/admin/gateway/learned"
RESET_URL = f"{BASE_URL}/admin/gateway/learned/reset"
SYNC_SECONDS = 1.5
def _seed(provider, model, method):
learned_store.remember(provider, model, method)
time.sleep(SYNC_SECONDS)
def test_learned_endpoints_require_admin(seeded_db):
assert requests.get(LIST_URL, headers=JSON_gateway, allow_redirects=False).status_code == 401
assert (
requests.post(RESET_URL, json={}, headers=JSON_gateway, allow_redirects=False).status_code
== 401
)
key = member_key()
member = {**JSON_gateway, "X-API-KEY": key}
assert requests.get(LIST_URL, headers=member, allow_redirects=False).status_code == 403
assert (
requests.post(RESET_URL, json={}, headers=member, allow_redirects=False).status_code == 403
)
def test_admin_lists_a_learned_method(seeded_db):
provider = _unique_gateway("lprov")
_seed(provider, "vendor/learned", ShapeMethod("none", ("stream_options",)))
try:
payload = admin_session(seeded_db).get(LIST_URL).json()
row = next(r for r in payload["learned"] if r["provider"] == provider)
assert row["target_model"] == "vendor/learned"
assert row["thinking_dialect"] == "none"
assert row["drop_keys"] == ["stream_options"]
assert set(payload["stats"]) == {"resolved", "exhausted", "healed"}
assert payload["count"] >= 1
finally:
learned_store.forget(provider, "vendor/learned")
def test_admin_resets_a_learned_method_and_it_is_audited(seeded_db):
provider = _unique_gateway("lprov")
_seed(provider, "vendor/reset", ShapeMethod("none"))
admin = admin_session(seeded_db)
response = admin.post(RESET_URL, json={"provider": provider, "target_model": "vendor/reset"})
assert response.status_code == 200
assert response.json()["ok"] is True
refresh_snapshot()
entries = admin.get(
f"{BASE_URL}/admin/audit-log",
headers=JSON_gateway,
params={"event_key": "gateway.learned.reset"},
).json()["entries"]
assert any(e.get("target_type") == "gateway_learned_method" for e in entries)
again = admin.post(RESET_URL, json={"provider": provider, "target_model": "vendor/reset"})
assert again.status_code == 404
def test_reset_validates_the_target_model(seeded_db):
response = admin_session(seeded_db).post(RESET_URL, json={"provider": "x"})
assert response.status_code in (400, 422)
+47
View File
@@ -299,3 +299,50 @@ def test_image_test_opens_result_modal(page, seeded_db):
httpd.shutdown()
requests.delete(f"{BASE_URL}/admin/gateway/models/{source}", headers=auth)
requests.delete(f"{BASE_URL}/admin/gateway/providers/{provider}", headers=auth)
def test_model_grid_shows_a_learned_shape_badge_and_reset(page, seeded_db):
import time
from devplacepy.services.openai_gateway.learned import learned_store
from devplacepy.services.openai_gateway.shape import ShapeMethod
user = seeded_db["alice"]
_promote_to_admin(user["username"])
key = _admin_api_key(user["username"])
auth = {"X-API-KEY": key}
provider = f"e2elearn{int(time.time() * 1000)}"
source = f"{provider}-src"
try:
requests.post(
f"{BASE_URL}/admin/gateway/providers",
json={"name": provider, "base_url": "http://127.0.0.1:9/v1/chat/completions"},
headers=auth,
)
requests.post(
f"{BASE_URL}/admin/gateway/models",
json={
"source_model": source,
"provider": provider,
"target_model": "vendor/learned",
"kind": "chat",
},
headers=auth,
)
learned_store.remember(provider, "vendor/learned", ShapeMethod("none"))
time.sleep(1.5)
login_user(page, user)
page.goto(f"{BASE_URL}/admin/gateway?tab=models", wait_until="domcontentloaded")
row = page.locator("table.admin-table tr", has_text=source)
row.wait_for(state="visible")
assert "shape: none" in (row.inner_text() or "")
row.get_by_role("button", name="Reset learned", exact=True).click()
page.locator(".dialog-overlay.visible .dialog-confirm").click()
page.wait_for_url(f"{BASE_URL}/admin/gateway?tab=models", wait_until="domcontentloaded")
row = page.locator("table.admin-table tr", has_text=source)
row.wait_for(state="visible")
assert "shape: none" not in (row.inner_text() or "")
finally:
learned_store.forget(provider, "vendor/learned")
requests.delete(f"{BASE_URL}/admin/gateway/models/{source}", headers=auth)
requests.delete(f"{BASE_URL}/admin/gateway/providers/{provider}", headers=auth)
@@ -3441,3 +3441,240 @@ def test_native_route_keeps_forwarding_tools(local_db, monkeypatch):
assert "<tools>" not in sent["instructions"]
finally:
_teardown_responses_route_openai_gateway()
VAIDS_REJECTION_openai_gateway = {
"details": {"_errors": ["Unrecognized key(s) in object: 'thinking'"]},
"error": "Invalid request parameters",
"issues": [
{
"code": "unrecognized_keys",
"keys": ["thinking"],
"path": [],
"message": "Unrecognized key(s) in object: 'thinking'",
}
],
}
def _rejection_openai_gateway(key):
return {
"details": {"_errors": [f"Unrecognized key(s) in object: '{key}'"]},
"error": "Invalid request parameters",
"issues": [{"code": "unrecognized_keys", "keys": [key], "path": []}],
}
class FakeStrictClient_openai_gateway:
rejected = ("thinking",)
def __init__(self, *a, **k):
self.calls = []
def build_request(self, method, url, headers=None, json=None, content=None):
return FakeRequest(method, url, json)
async def send(self, request, stream=False):
body = request.json_body or {}
self.calls.append((request.url, dict(body)))
for key in self.rejected:
if key in body:
return FakeResp_openai_gateway(
status=400, payload=_rejection_openai_gateway(key)
)
return FakeResp_openai_gateway(
payload={
"id": "x",
"model": body.get("model"),
"choices": [{"message": {"content": "hi there"}}],
}
)
async def aclose(self):
pass
class FakeStubbornClient_openai_gateway(FakeStrictClient_openai_gateway):
async def send(self, request, stream=False):
self.calls.append((request.url, dict(request.json_body or {})))
return FakeResp_openai_gateway(
status=400, payload=_rejection_openai_gateway("thinking")
)
class FakeReasoningStrictClient_openai_gateway(FakeStrictClient_openai_gateway):
rejected = ("reasoning",)
def _shape_cfg_openai_gateway():
svc = GatewayService()
cfg = svc.effective_config()
cfg["gateway_vision_enabled"] = False
cfg["gateway_max_retries"] = 0
return svc, cfg
def _chat_openai_gateway(rt, cfg, owner, stream=False):
body = {"messages": [{"role": "user", "content": "hi"}]}
if stream:
body["stream"] = True
return run_async(rt.handle_chat(body, cfg, ("guest", owner), "test", "default"))
def test_chat_auto_resolves_a_rejected_thinking_key_and_remembers_it(local_db, monkeypatch):
from devplacepy.services.openai_gateway.learned import learned_store
monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeStrictClient_openai_gateway)
svc, cfg = _shape_cfg_openai_gateway()
model = cfg["gateway_model"]
learned_store.forget("default", model)
try:
rt = svc.runtime()
response = _chat_openai_gateway(rt, cfg, "shape_resolve_first")
assert response.status_code == 200
assert len(rt._client.calls) == 2
assert "thinking" in rt._client.calls[0][1]
assert "thinking" not in rt._client.calls[1][1]
learned = learned_store.get("default", model)
assert learned is not None and learned.thinking_dialect == "none"
row = get_table("gateway_usage_ledger").find_one(owner_id="shape_resolve_first")
assert row["resolved_method"] == "none"
assert row["success"] == 1
second = _chat_openai_gateway(rt, cfg, "shape_resolve_second")
assert second.status_code == 200
assert len(rt._client.calls) == 3
assert "thinking" not in rt._client.calls[2][1]
finally:
learned_store.forget("default", model)
def test_chat_auto_resolve_works_for_streams(local_db, monkeypatch):
from devplacepy.services.openai_gateway.learned import learned_store
monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeStrictClient_openai_gateway)
svc, cfg = _shape_cfg_openai_gateway()
model = cfg["gateway_model"]
learned_store.forget("default", model)
try:
rt = svc.runtime()
response = _chat_openai_gateway(rt, cfg, "shape_resolve_stream", stream=True)
async def drain():
out = []
async for chunk in response.body_iterator:
out.append(chunk if isinstance(chunk, str) else chunk.decode())
return "".join(out)
assert "[DONE]" in run_async(drain())
assert len(rt._client.calls) == 2
assert learned_store.get("default", model) is not None
finally:
learned_store.forget("default", model)
def test_chat_genuine_bad_request_is_not_auto_resolved(local_db, monkeypatch):
from devplacepy.services.openai_gateway.learned import learned_store
monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeBadRequestClient_openai_gateway)
svc, cfg = _shape_cfg_openai_gateway()
model = cfg["gateway_model"]
learned_store.forget("default", model)
rt = svc.runtime()
response = _chat_openai_gateway(rt, cfg, "shape_genuine_bad")
assert response.status_code == 400
assert len(rt._client.calls) == 1
assert learned_store.get("default", model) is None
def test_chat_exhausted_resolution_returns_the_original_error_and_stores_nothing(
local_db, monkeypatch
):
from devplacepy.services.openai_gateway.learned import learned_store
from devplacepy.services.openai_gateway.shape import MAX_RESOLVE_ATTEMPTS
monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeStubbornClient_openai_gateway)
svc, cfg = _shape_cfg_openai_gateway()
model = cfg["gateway_model"]
learned_store.forget("default", model)
rt = svc.runtime()
response = _chat_openai_gateway(rt, cfg, "shape_exhausted")
assert response.status_code == 400
assert "thinking" in response.body.decode()
assert len(rt._client.calls) <= 1 + MAX_RESOLVE_ATTEMPTS
assert learned_store.get("default", model) is None
def test_chat_rejected_learned_method_is_forgotten_and_ladder_restarts(local_db, monkeypatch):
from devplacepy.services.openai_gateway.learned import learned_store
from devplacepy.services.openai_gateway.shape import ShapeMethod
monkeypatch.setattr(
gwmod.httpx, "AsyncClient", FakeReasoningStrictClient_openai_gateway
)
svc, cfg = _shape_cfg_openai_gateway()
model = cfg["gateway_model"]
learned_store.remember("default", model, ShapeMethod("openrouter"))
try:
rt = svc.runtime()
response = _chat_openai_gateway(rt, cfg, "shape_heal")
assert response.status_code == 200
assert "reasoning" in rt._client.calls[0][1]
assert "thinking" in rt._client.calls[1][1]
assert learned_store.get("default", model) is None
finally:
learned_store.forget("default", model)
def test_fallback_attempt_is_shaped_with_its_own_learned_method(local_db, monkeypatch):
from devplacepy.services.openai_gateway import routing
from devplacepy.services.openai_gateway.learned import learned_store
from devplacepy.services.openai_gateway.shape import ShapeMethod
class FakeFallbackShapeClient(FakeStrictClient_openai_gateway):
rejected = ()
async def send(self, request, stream=False):
body = request.json_body or {}
self.calls.append((request.url, dict(body)))
if body.get("model") == "shape-primary-target":
return FakeResp_openai_gateway(status=503, payload={"error": {"message": "down"}})
return FakeResp_openai_gateway(
payload={
"id": "x",
"model": body.get("model"),
"choices": [{"message": {"content": "ok"}}],
}
)
monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeFallbackShapeClient)
routing.model_store.set(
routing.ModelRouteIn(source_model="shape-backup", target_model="shape-backup-target")
)
routing.model_store.set(
routing.ModelRouteIn(
source_model="shape-primary",
target_model="shape-primary-target",
fallback_model="shape-backup",
)
)
learned_store.remember("default", "shape-backup-target", ShapeMethod("none"))
try:
svc, cfg = _shape_cfg_openai_gateway()
rt = svc.runtime()
response = run_async(
rt.handle_chat(
{"model": "shape-primary", "messages": [{"role": "user", "content": "hi"}]},
cfg,
("guest", "shape_fallback"),
"test",
"default",
)
)
assert response.status_code == 200
assert "thinking" in rt._client.calls[0][1]
assert rt._client.calls[-1][1]["model"] == "shape-backup-target"
assert "thinking" not in rt._client.calls[-1][1]
finally:
learned_store.forget("default", "shape-backup-target")
routing.model_store.remove("shape-primary")
routing.model_store.remove("shape-backup")
@@ -0,0 +1,90 @@
# retoor <retoor@molodetz.nl>
from devplacepy.services.openai_gateway import learned
from devplacepy.services.openai_gateway.learned import learned_store
from devplacepy.services.openai_gateway.shape import NATIVE_METHOD, ShapeMethod
def test_remember_get_list_and_forget(local_db):
learned_store.forget("lt-prov", "lt-model")
method = ShapeMethod("none", ("stream_options",))
learned_store.remember("LT-Prov", "lt-model", method)
try:
assert learned_store.get("lt-prov", "lt-model") == method
rows = [r for r in learned_store.list() if r["target_model"] == "lt-model"]
assert rows[0]["provider"] == "lt-prov"
assert rows[0]["drop_keys"] == ["stream_options"]
assert learned_store.forget("lt-prov", "lt-model") is True
assert learned_store.get("lt-prov", "lt-model") is None
assert learned_store.forget("lt-prov", "lt-model") is False
finally:
learned_store.forget("lt-prov", "lt-model")
def test_blank_provider_maps_to_default_and_native_is_never_stored(local_db):
learned_store.forget("default", "lt-default-model")
learned_store.remember("", "lt-default-model", NATIVE_METHOD)
assert learned_store.get("default", "lt-default-model") is None
learned_store.remember("", "lt-default-model", ShapeMethod("none"))
try:
assert learned_store.get("default", "lt-default-model") == ShapeMethod("none")
finally:
learned_store.forget("default", "lt-default-model")
def test_remember_upserts_on_the_unique_route(local_db):
learned_store.forget("lt-up", "lt-m")
try:
learned_store.remember("lt-up", "lt-m", ShapeMethod("none"))
learned_store.remember("lt-up", "lt-m", ShapeMethod("openrouter"))
assert learned_store.get("lt-up", "lt-m") == ShapeMethod("openrouter")
assert len([r for r in learned_store.list() if r["provider"] == "lt-up"]) == 1
finally:
learned_store.forget("lt-up", "lt-m")
def test_forget_provider_clears_only_that_provider(local_db):
learned_store.remember("lt-fp", "a", ShapeMethod("none"))
learned_store.remember("lt-fp", "b", ShapeMethod("none"))
learned_store.remember("lt-other", "a", ShapeMethod("none"))
try:
assert learned_store.forget_provider("lt-fp") == 2
assert learned_store.get("lt-fp", "a") is None
assert learned_store.get("lt-other", "a") is not None
finally:
learned_store.forget_provider("lt-fp")
learned_store.forget_provider("lt-other")
def test_row_cap_replaces_the_oldest(local_db, monkeypatch):
monkeypatch.setattr(learned, "MAX_LEARNED_ROWS", 2)
keys = [("lt-cap", f"m{i}") for i in range(3)]
try:
for provider, model in keys:
learned_store.remember(provider, model, ShapeMethod("none"))
present = [k for k in keys if learned_store.get(*k) is not None]
assert len(present) <= 2
assert learned_store.get(*keys[2]) is not None
finally:
learned_store.forget_provider("lt-cap")
def test_provider_endpoint_edit_forgets_learned_methods(local_db):
from devplacepy.services.openai_gateway import routing
routing.provider_store.set(
routing.ProviderIn(name="lt-edit", base_url="https://a.example/v1/chat/completions")
)
learned_store.remember("lt-edit", "m", ShapeMethod("none"))
try:
routing.provider_store.set(
routing.ProviderIn(name="lt-edit", base_url="https://a.example/v1/chat/completions")
)
assert learned_store.get("lt-edit", "m") is not None
routing.provider_store.set(
routing.ProviderIn(name="lt-edit", base_url="https://b.example/v1/chat/completions")
)
assert learned_store.get("lt-edit", "m") is None
finally:
routing.provider_store.remove("lt-edit")
learned_store.forget_provider("lt-edit")
+174
View File
@@ -0,0 +1,174 @@
# retoor <retoor@molodetz.nl>
import json
from devplacepy.services.openai_gateway.shape import (
MAX_RESOLVE_ATTEMPTS,
NATIVE_METHOD,
RejectedKeys,
ShapeMethod,
ShapeResolver,
ShapeRetrier,
)
VAIDS_BODY = json.dumps(
{
"details": {"_errors": ["Unrecognized key(s) in object: 'thinking'"]},
"error": "Invalid request parameters",
"issues": [
{
"code": "unrecognized_keys",
"keys": ["thinking"],
"path": [],
"message": "Unrecognized key(s) in object: 'thinking'",
}
],
}
)
class MemoryStore_openai_gateway:
def __init__(self):
self.rows = {}
self.forgotten = []
def get(self, provider, model):
return self.rows.get((provider, model))
def remember(self, provider, model, method):
self.rows[(provider, model)] = method
def forget(self, provider, model):
self.forgotten.append((provider, model))
return self.rows.pop((provider, model), None) is not None
def test_parse_real_vaids_body():
assert RejectedKeys.parse(VAIDS_BODY) == ("thinking",)
def test_parse_openai_style_bodies():
assert RejectedKeys.parse(
'{"error":{"message":"Unrecognized request argument supplied: foo","param":null}}'
) == ("foo",)
assert RejectedKeys.parse(
'{"error":{"message":"Unknown parameter: \'stream_options\'.",'
'"param":"stream_options","code":"unknown_parameter"}}'
) == ("stream_options",)
def test_parse_serde_style_and_plain_text():
assert RejectedKeys.parse("unknown field `thinking`, expected one of `model`") == (
"thinking",
)
def test_parse_returns_empty_for_a_genuine_bad_request():
assert RejectedKeys.parse('{"error":{"message":"invalid request: bad param"}}') == ()
assert RejectedKeys.parse("") == ()
assert RejectedKeys.parse(None) == ()
def test_resolver_prefers_none_when_thinking_is_rejected():
resolver = ShapeResolver("deepseek", "chat_completions")
method = resolver.next_method({NATIVE_METHOD}, NATIVE_METHOD, ("thinking",))
assert method == ShapeMethod("none")
def test_resolver_skips_dialects_that_emit_the_rejected_key():
resolver = ShapeResolver("deepseek", "chat_completions")
tried = {NATIVE_METHOD, ShapeMethod("none")}
method = resolver.next_method(tried, ShapeMethod("none"), ("reasoning",))
assert method == ShapeMethod("deepseek")
def test_resolver_adds_a_droppable_key():
resolver = ShapeResolver("deepseek", "chat_completions")
method = resolver.next_method({NATIVE_METHOD}, NATIVE_METHOD, ("stream_options",))
assert method == ShapeMethod("native", ("stream_options",))
def test_resolver_refuses_unknown_and_protected_keys():
resolver = ShapeResolver("deepseek", "chat_completions")
for key in ("messages", "model", "tools", "something_else"):
assert resolver.next_method({NATIVE_METHOD}, NATIVE_METHOD, (key,)) is None
def test_resolver_responses_ladder_is_reasoning_only():
resolver = ShapeResolver("deepseek", "responses")
method = resolver.next_method({NATIVE_METHOD}, NATIVE_METHOD, ("reasoning",))
assert method == ShapeMethod("none")
assert resolver.next_method({NATIVE_METHOD}, NATIVE_METHOD, ("thinking",)) is None
def test_method_json_round_trip_and_validation():
method = ShapeMethod("none", ("stream_options",))
assert ShapeMethod.from_json(method.to_json()) == method
assert ShapeMethod.from_json('{"thinking_dialect":"bogus","drop_keys":[]}') is None
assert ShapeMethod.from_json('{"thinking_dialect":"none","drop_keys":["tools"]}') is None
assert ShapeMethod.from_json("not json") is None
def test_apply_none_clears_thinking_and_native_writes_the_dialect():
cfg = {"gateway_upstream_url": "https://strict.example/v1/chat/completions"}
payload = {"model": "m", "messages": [], "thinking": {"type": "enabled"}}
native = NATIVE_METHOD.apply(payload, cfg, "chat_completions", False, lambda m: None)
assert native["thinking"] == {"type": "enabled"}
none = ShapeMethod("none").apply(payload, cfg, "chat_completions", False, lambda m: None)
assert "thinking" not in none and "reasoning" not in none
assert payload["thinking"] == {"type": "enabled"}
def test_apply_drops_stream_options_when_learned():
cfg = {"gateway_upstream_url": "https://strict.example/v1/chat/completions"}
payload = {"model": "m", "messages": [], "stream": True}
kept = NATIVE_METHOD.apply(payload, cfg, "chat_completions", True, lambda m: None)
assert kept["stream_options"] == {"include_usage": True}
dropped = ShapeMethod("native", ("stream_options",)).apply(
payload, cfg, "chat_completions", True, lambda m: None
)
assert "stream_options" not in dropped
def test_strip_wire_removes_reasoning_for_responses_none():
body = {"model": "m", "reasoning": {"effort": "minimal"}, "input": []}
stripped = ShapeMethod("none").strip_wire(body, "responses")
assert "reasoning" not in stripped and body["reasoning"]
assert NATIVE_METHOD.strip_wire(body, "responses") is body
def test_retrier_success_remembers_and_native_success_does_not():
store = MemoryStore_openai_gateway()
retrier = ShapeRetrier.for_route(store, "p", "m", "chat_completions", "deepseek", lambda m: None)
retrier.succeeded()
assert store.rows == {}
method = retrier.next_after(400, VAIDS_BODY)
assert method == ShapeMethod("none")
retrier.succeeded()
assert store.rows[("p", "m")] == ShapeMethod("none")
def test_retrier_ignores_non_shape_statuses_and_unparsable_bodies():
store = MemoryStore_openai_gateway()
retrier = ShapeRetrier.for_route(store, "p", "m", "chat_completions", "deepseek", lambda m: None)
assert retrier.next_after(500, VAIDS_BODY) is None
assert retrier.next_after(400, '{"error":"nope"}') is None
def test_retrier_attempts_are_capped():
store = MemoryStore_openai_gateway()
retrier = ShapeRetrier.for_route(store, "p", "m", "chat_completions", "deepseek", lambda m: None)
retrier.attempts = MAX_RESOLVE_ATTEMPTS
assert retrier.next_after(400, VAIDS_BODY) is None
def test_retrier_forgets_a_rejected_learned_method_once():
store = MemoryStore_openai_gateway()
store.rows[("p", "m")] = ShapeMethod("openrouter")
retrier = ShapeRetrier.for_route(store, "p", "m", "chat_completions", "deepseek", lambda m: None)
assert retrier.method == ShapeMethod("openrouter")
method = retrier.next_after(
400, '{"issues":[{"code":"unrecognized_keys","keys":["reasoning"]}]}'
)
assert method == NATIVE_METHOD
assert store.forgotten == [("p", "m")]