forked from retoor/devplacepy
Add OpenCode Zen support, model health/stats dashboard, and gateway fallback fixes
AI gateway: - Add a generic, admin-selectable `client_profile` field on gateway_providers (e.g. "opencode") so a provider needing special request headers (OpenCode Zen's client-identity spoofing) is configured like any other provider, not hardcoded by name. - Track per-(provider, model) reliability/speed/latency health in memory, seeded from the existing gateway_usage_ledger at startup - purely observational, never influences routing. - New Stats tab on /admin/gateway: request volume, latency, per-model breakdowns, and reliability weight, charted with a vendored Chart.js and devplace's own theme tokens. - Record which model a failed request actually fell back to (fallback_used_route), surfaced in the Recent Failures table. - Stop excluding context_length errors from fallback, and skip a primary attempt outright when its known context window is already too small for the estimated request size, going straight to the fallback. - gateway_usage_ledger's provider/fallback_used_route columns and indexes are ensured centrally in database/schema.py's init_db(), the single point of truth for this table's schema. - Non-OpenAI upstream routing and client-model passthrough; trust only the upstream's own X-Gateway-Model header for served-model attribution. Devii agent: - Fix a real lockup: plan/verify tools could be individually disabled via the admin tool toggles while still being required by the protocol gate, permanently bricking any task that needed tools. They can no longer be disabled, and the gate now also checks the tool is actually offered. - Fix compaction being silently calibrated for a 1M-token model while running a much smaller one: context budget is now percentage-based and the summarizer's own request is sized to fit the real model. - Give a specific, actionable retry message when plan()'s own arguments get cut off by the output limit, and tighten its schema to discourage overlong plans. Other: - Backup service: offload completed backups to a remote Hetzner Storage Box. - Container manager: fix orphan blob leaks from sync races, add a two-phase plan/execute `system prune` CLI command. - Admin gateway UI: replace the JS-rendered model/provider tables with server-rendered forms and pages. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01DhmEkvutuwtzFVcLbTrhdo
This commit is contained in:
Claude Sonnet 5
@@ -66,6 +66,32 @@ def test_gateway_page_requires_admin(seeded_db):
|
||||
|
||||
def test_gateway_page_renders_for_admin(seeded_db):
|
||||
admin = admin_session(seeded_db)
|
||||
response = admin.get(f"{BASE_URL}/admin/gateway", headers={})
|
||||
response = admin.get(f"{BASE_URL}/admin/gateway", headers={"Accept": "text/html"})
|
||||
assert response.status_code == 200
|
||||
assert "Gateway routing" in response.text
|
||||
|
||||
|
||||
def test_gateway_page_tabs_default_and_select_content(seeded_db):
|
||||
admin = admin_session(seeded_db)
|
||||
|
||||
default_page = admin.get(f"{BASE_URL}/admin/gateway", headers={"Accept": "text/html"})
|
||||
assert "Model routes" in default_page.text
|
||||
assert "GatewayAdmin" not in default_page.text
|
||||
|
||||
providers_page = admin.get(f"{BASE_URL}/admin/gateway?tab=providers", headers={"Accept": "text/html"})
|
||||
assert "Providers" in providers_page.text
|
||||
assert "Model routes" not in providers_page.text
|
||||
|
||||
quota_page = admin.get(f"{BASE_URL}/admin/gateway?tab=quota", headers={"Accept": "text/html"})
|
||||
assert "Quota rules" in quota_page.text
|
||||
assert "Model routes" not in quota_page.text
|
||||
|
||||
unknown_tab_page = admin.get(f"{BASE_URL}/admin/gateway?tab=bogus", headers={"Accept": "text/html"})
|
||||
assert "Model routes" in unknown_tab_page.text
|
||||
|
||||
|
||||
def test_gateway_page_json_reports_active_tab(seeded_db):
|
||||
admin = admin_session(seeded_db)
|
||||
response = admin.get(f"{BASE_URL}/admin/gateway?tab=providers")
|
||||
assert response.status_code == 200
|
||||
assert response.json()["tab"] == "providers"
|
||||
|
||||
@@ -224,6 +224,113 @@ def test_model_route_fallback_rejects_self_reference(seeded_db):
|
||||
assert response.json()["ok"] is False
|
||||
|
||||
|
||||
def test_model_form_pages_require_admin(seeded_db):
|
||||
assert (
|
||||
requests.get(
|
||||
f"{BASE_URL}/admin/gateway/models/new",
|
||||
headers=JSON_gateway,
|
||||
allow_redirects=False,
|
||||
).status_code
|
||||
== 401
|
||||
)
|
||||
key = member_key()
|
||||
assert (
|
||||
requests.get(
|
||||
f"{BASE_URL}/admin/gateway/models/new",
|
||||
headers={**JSON_gateway, "X-API-KEY": key},
|
||||
allow_redirects=False,
|
||||
).status_code
|
||||
== 403
|
||||
)
|
||||
|
||||
|
||||
def test_model_add_edit_delete_via_page(seeded_db):
|
||||
admin = admin_session(seeded_db)
|
||||
source = _unique_gateway("modelpage")
|
||||
|
||||
new_page = admin.get(f"{BASE_URL}/admin/gateway/models/new")
|
||||
assert new_page.status_code == 200
|
||||
assert new_page.json()["is_edit"] is False
|
||||
|
||||
created = admin.post(
|
||||
f"{BASE_URL}/admin/gateway/models/new",
|
||||
data={
|
||||
"source_model": source,
|
||||
"target_model": "vendor/page",
|
||||
"kind": "chat",
|
||||
"price_output_per_m": "1.5",
|
||||
"off_peak_start": "22:30",
|
||||
"off_peak_end": "06:00",
|
||||
"off_peak_discount_pct": "10",
|
||||
"is_active": "1",
|
||||
},
|
||||
allow_redirects=False,
|
||||
)
|
||||
assert created.status_code == 302
|
||||
assert created.headers["location"] == "/admin/gateway?tab=models"
|
||||
|
||||
edit_page = admin.get(f"{BASE_URL}/admin/gateway/models/{source}/edit")
|
||||
assert edit_page.status_code == 200
|
||||
form = edit_page.json()["form"]
|
||||
assert form["target_model"] == "vendor/page"
|
||||
assert form["off_peak_start"] == "22:30"
|
||||
assert form["off_peak_end"] == "06:00"
|
||||
|
||||
edit_html = admin.get(f"{BASE_URL}/admin/gateway/models/{source}/edit", headers={"Accept": "text/html"})
|
||||
assert f'value="{source}"' in edit_html.text
|
||||
assert "readonly" in edit_html.text
|
||||
|
||||
updated = admin.post(
|
||||
f"{BASE_URL}/admin/gateway/models/{source}/edit",
|
||||
data={"target_model": "vendor/page2", "kind": "chat", "is_active": "1"},
|
||||
allow_redirects=False,
|
||||
)
|
||||
assert updated.status_code == 302
|
||||
|
||||
relisted = admin.get(f"{BASE_URL}/admin/gateway/models").json()
|
||||
row = next(m for m in relisted["models"] if m["source_model"] == source)
|
||||
assert row["target_model"] == "vendor/page2"
|
||||
|
||||
deleted = admin.post(
|
||||
f"{BASE_URL}/admin/gateway/models/{source}/delete", allow_redirects=False
|
||||
)
|
||||
assert deleted.status_code == 302
|
||||
assert admin.get(f"{BASE_URL}/admin/gateway/models/{source}/edit").status_code == 404
|
||||
|
||||
|
||||
def test_model_page_fallback_must_be_the_same_kind(seeded_db):
|
||||
admin = admin_session(seeded_db)
|
||||
embed_route = _unique_gateway("fbpage-embed")
|
||||
chat_route = _unique_gateway("fbpage-chat")
|
||||
|
||||
admin.post(
|
||||
f"{BASE_URL}/admin/gateway/models",
|
||||
json={"source_model": embed_route, "target_model": "vendor/e", "kind": "embed"},
|
||||
)
|
||||
response = admin.post(
|
||||
f"{BASE_URL}/admin/gateway/models/new",
|
||||
data={
|
||||
"source_model": chat_route,
|
||||
"target_model": "vendor/c",
|
||||
"kind": "chat",
|
||||
"fallback_model": embed_route,
|
||||
},
|
||||
)
|
||||
assert response.status_code == 400
|
||||
assert "message" in response.json()["error"]
|
||||
|
||||
admin.delete(f"{BASE_URL}/admin/gateway/models/{embed_route}")
|
||||
|
||||
|
||||
def test_model_edit_page_404_for_missing_model(seeded_db):
|
||||
admin = admin_session(seeded_db)
|
||||
missing = _unique_gateway("ghostmodel")
|
||||
assert admin.get(f"{BASE_URL}/admin/gateway/models/{missing}/edit").status_code == 404
|
||||
assert (
|
||||
admin.post(f"{BASE_URL}/admin/gateway/models/{missing}/delete").status_code == 404
|
||||
)
|
||||
|
||||
|
||||
def test_model_route_validation(seeded_db):
|
||||
admin = admin_session(seeded_db)
|
||||
missing_target = admin.post(
|
||||
|
||||
@@ -0,0 +1,120 @@
|
||||
# retoor <retoor@molodetz.nl>
|
||||
|
||||
import http.server
|
||||
import json
|
||||
import socket
|
||||
import socketserver
|
||||
import threading
|
||||
|
||||
import requests
|
||||
|
||||
from tests.conftest import BASE_URL
|
||||
from tests.api.admin.gateway.index import (
|
||||
JSON_gateway,
|
||||
admin_session,
|
||||
member_key,
|
||||
_unique_gateway,
|
||||
)
|
||||
|
||||
|
||||
def _fake_upstream(status_code, payload):
|
||||
sock = socket.socket()
|
||||
sock.bind(("127.0.0.1", 0))
|
||||
port = sock.getsockname()[1]
|
||||
sock.close()
|
||||
body = json.dumps(payload).encode() if payload is not None else b""
|
||||
seen_auth = {}
|
||||
|
||||
class Handler(http.server.BaseHTTPRequestHandler):
|
||||
def do_GET(self):
|
||||
seen_auth["path"] = self.path
|
||||
seen_auth["authorization"] = self.headers.get("Authorization")
|
||||
self.send_response(status_code)
|
||||
self.send_header("Content-Type", "application/json")
|
||||
self.end_headers()
|
||||
self.wfile.write(body)
|
||||
|
||||
def log_message(self, *args):
|
||||
pass
|
||||
|
||||
httpd = socketserver.TCPServer(("127.0.0.1", port), Handler)
|
||||
thread = threading.Thread(target=httpd.serve_forever, daemon=True)
|
||||
thread.start()
|
||||
return httpd, port, seen_auth
|
||||
|
||||
|
||||
def test_provider_models_requires_admin(seeded_db):
|
||||
assert (
|
||||
requests.get(
|
||||
f"{BASE_URL}/admin/gateway/provider-models",
|
||||
headers=JSON_gateway,
|
||||
allow_redirects=False,
|
||||
).status_code
|
||||
== 401
|
||||
)
|
||||
key = member_key()
|
||||
assert (
|
||||
requests.get(
|
||||
f"{BASE_URL}/admin/gateway/provider-models",
|
||||
headers={**JSON_gateway, "X-API-KEY": key},
|
||||
allow_redirects=False,
|
||||
).status_code
|
||||
== 403
|
||||
)
|
||||
|
||||
|
||||
def test_provider_models_returns_the_list_on_success(seeded_db):
|
||||
admin = admin_session(seeded_db)
|
||||
name = _unique_gateway("modelsupstream").lower()
|
||||
httpd, port, seen = _fake_upstream(
|
||||
200, {"data": [{"id": "vendor/a"}, {"id": "vendor/b"}]}
|
||||
)
|
||||
try:
|
||||
admin.post(
|
||||
f"{BASE_URL}/admin/gateway/providers",
|
||||
json={
|
||||
"name": name,
|
||||
"base_url": f"http://127.0.0.1:{port}/v1/chat/completions",
|
||||
"api_key": "sk-fake",
|
||||
},
|
||||
)
|
||||
response = admin.get(
|
||||
f"{BASE_URL}/admin/gateway/provider-models", params={"provider": name}
|
||||
)
|
||||
assert response.status_code == 200
|
||||
assert response.json()["models"] == ["vendor/a", "vendor/b"]
|
||||
assert seen["path"] == "/v1/models"
|
||||
assert seen["authorization"] == "Bearer sk-fake"
|
||||
finally:
|
||||
httpd.shutdown()
|
||||
admin.delete(f"{BASE_URL}/admin/gateway/providers/{name}")
|
||||
|
||||
|
||||
def test_provider_models_404_when_upstream_has_no_model_listing(seeded_db):
|
||||
admin = admin_session(seeded_db)
|
||||
name = _unique_gateway("modelsupstream404").lower()
|
||||
httpd, port, _ = _fake_upstream(404, {})
|
||||
try:
|
||||
admin.post(
|
||||
f"{BASE_URL}/admin/gateway/providers",
|
||||
json={
|
||||
"name": name,
|
||||
"base_url": f"http://127.0.0.1:{port}/v1/chat/completions",
|
||||
},
|
||||
)
|
||||
response = admin.get(
|
||||
f"{BASE_URL}/admin/gateway/provider-models", params={"provider": name}
|
||||
)
|
||||
assert response.status_code == 404
|
||||
finally:
|
||||
httpd.shutdown()
|
||||
admin.delete(f"{BASE_URL}/admin/gateway/providers/{name}")
|
||||
|
||||
|
||||
def test_provider_models_404_for_unknown_provider(seeded_db):
|
||||
admin = admin_session(seeded_db)
|
||||
response = admin.get(
|
||||
f"{BASE_URL}/admin/gateway/provider-models",
|
||||
params={"provider": _unique_gateway("ghostprovider")},
|
||||
)
|
||||
assert response.status_code == 404
|
||||
@@ -82,3 +82,115 @@ def test_provider_name_validation(seeded_db):
|
||||
)
|
||||
assert bad.status_code == 400
|
||||
assert bad.json()["ok"] is False
|
||||
|
||||
|
||||
def test_provider_form_pages_require_admin(seeded_db):
|
||||
name = _unique_gateway("provpage").lower()
|
||||
assert (
|
||||
requests.get(
|
||||
f"{BASE_URL}/admin/gateway/providers/new",
|
||||
headers=JSON_gateway,
|
||||
allow_redirects=False,
|
||||
).status_code
|
||||
== 401
|
||||
)
|
||||
key = member_key()
|
||||
assert (
|
||||
requests.get(
|
||||
f"{BASE_URL}/admin/gateway/providers/new",
|
||||
headers={**JSON_gateway, "X-API-KEY": key},
|
||||
allow_redirects=False,
|
||||
).status_code
|
||||
== 403
|
||||
)
|
||||
assert (
|
||||
requests.post(
|
||||
f"{BASE_URL}/admin/gateway/providers/new",
|
||||
data={"name": name},
|
||||
headers={**JSON_gateway, "X-API-KEY": key},
|
||||
allow_redirects=False,
|
||||
).status_code
|
||||
== 403
|
||||
)
|
||||
|
||||
|
||||
def test_provider_add_edit_delete_via_page(seeded_db):
|
||||
admin = admin_session(seeded_db)
|
||||
name = _unique_gateway("provpage").lower()
|
||||
|
||||
new_page = admin.get(f"{BASE_URL}/admin/gateway/providers/new")
|
||||
assert new_page.status_code == 200
|
||||
assert new_page.json()["is_edit"] is False
|
||||
|
||||
created = admin.post(
|
||||
f"{BASE_URL}/admin/gateway/providers/new",
|
||||
data={
|
||||
"name": name,
|
||||
"base_url": "https://page.example/v1/chat/completions",
|
||||
"api_key": "sk-page",
|
||||
"is_active": "1",
|
||||
},
|
||||
allow_redirects=False,
|
||||
)
|
||||
assert created.status_code == 302
|
||||
assert created.headers["location"] == "/admin/gateway?tab=providers"
|
||||
|
||||
edit_page = admin.get(f"{BASE_URL}/admin/gateway/providers/{name}/edit")
|
||||
assert edit_page.status_code == 200
|
||||
edit_body = edit_page.json()
|
||||
assert edit_body["is_edit"] is True
|
||||
assert edit_body["form"]["base_url"] == "https://page.example/v1/chat/completions"
|
||||
|
||||
edit_html = admin.get(f"{BASE_URL}/admin/gateway/providers/{name}/edit", headers={"Accept": "text/html"})
|
||||
assert f'value="{name}"' in edit_html.text
|
||||
assert "readonly" in edit_html.text
|
||||
|
||||
updated = admin.post(
|
||||
f"{BASE_URL}/admin/gateway/providers/{name}/edit",
|
||||
data={"base_url": "https://page2.example/v1/chat/completions", "is_active": "0"},
|
||||
allow_redirects=False,
|
||||
)
|
||||
assert updated.status_code == 302
|
||||
|
||||
relisted = admin.get(f"{BASE_URL}/admin/gateway/providers").json()
|
||||
match = next(p for p in relisted["providers"] if p["name"] == name)
|
||||
assert match["base_url"] == "https://page2.example/v1/chat/completions"
|
||||
assert match["is_active"] is False
|
||||
|
||||
deleted = admin.post(
|
||||
f"{BASE_URL}/admin/gateway/providers/{name}/delete", allow_redirects=False
|
||||
)
|
||||
assert deleted.status_code == 302
|
||||
assert admin.get(f"{BASE_URL}/admin/gateway/providers/{name}/edit").status_code == 404
|
||||
|
||||
|
||||
def test_provider_page_validation_error_rerenders_with_message(seeded_db):
|
||||
admin = admin_session(seeded_db)
|
||||
response = admin.post(
|
||||
f"{BASE_URL}/admin/gateway/providers/new",
|
||||
data={"name": "has spaces!"},
|
||||
)
|
||||
assert response.status_code == 400
|
||||
assert "message" in response.json()["error"]
|
||||
|
||||
|
||||
def test_provider_page_validation_error_shows_banner_in_html(seeded_db):
|
||||
admin = admin_session(seeded_db)
|
||||
response = admin.post(
|
||||
f"{BASE_URL}/admin/gateway/providers/new",
|
||||
data={"name": "has spaces!"},
|
||||
headers={"Accept": "text/html"},
|
||||
)
|
||||
assert response.status_code == 400
|
||||
assert "gw-error" in response.text
|
||||
assert "letters, numbers, hyphen, underscore" in response.text
|
||||
|
||||
|
||||
def test_provider_edit_page_404_for_missing_provider(seeded_db):
|
||||
admin = admin_session(seeded_db)
|
||||
missing = _unique_gateway("ghostprov").lower()
|
||||
assert admin.get(f"{BASE_URL}/admin/gateway/providers/{missing}/edit").status_code == 404
|
||||
assert (
|
||||
admin.post(f"{BASE_URL}/admin/gateway/providers/{missing}/delete").status_code
|
||||
== 404
|
||||
)
|
||||
|
||||
@@ -11,6 +11,7 @@ from tests.api.admin.gateway.index import (
|
||||
JSON_gateway,
|
||||
admin_session,
|
||||
member_key,
|
||||
_unique_gateway,
|
||||
)
|
||||
from tests.conftest import BASE_URL
|
||||
|
||||
@@ -160,3 +161,121 @@ def test_spend_recorded_after_a_reset_counts_again(seeded_db):
|
||||
)
|
||||
_burn(owner, "appa", 0.5)
|
||||
assert _spent(owner, "appa") == 0.5
|
||||
|
||||
|
||||
def test_quota_rule_form_pages_require_admin(seeded_db):
|
||||
assert (
|
||||
requests.get(
|
||||
f"{BASE_URL}/admin/gateway/quota-rules/new",
|
||||
headers=JSON_gateway,
|
||||
allow_redirects=False,
|
||||
).status_code
|
||||
== 401
|
||||
)
|
||||
key = member_key()
|
||||
assert (
|
||||
requests.get(
|
||||
f"{BASE_URL}/admin/gateway/quota-rules/new",
|
||||
headers={**JSON_gateway, "X-API-KEY": key},
|
||||
allow_redirects=False,
|
||||
).status_code
|
||||
== 403
|
||||
)
|
||||
|
||||
|
||||
def test_quota_rule_add_edit_delete_reset_via_page(seeded_db):
|
||||
admin = admin_session(seeded_db)
|
||||
owner = _owner()
|
||||
app_ref = _unique_gateway("qrapp").lower()
|
||||
|
||||
new_page = admin.get(f"{BASE_URL}/admin/gateway/quota-rules/new")
|
||||
assert new_page.status_code == 200
|
||||
assert new_page.json()["is_edit"] is False
|
||||
|
||||
created = admin.post(
|
||||
f"{BASE_URL}/admin/gateway/quota-rules/new",
|
||||
data={
|
||||
"owner_kind": "user",
|
||||
"owner_id": owner,
|
||||
"app_reference": app_ref,
|
||||
"limit_usd": "2.5",
|
||||
"is_active": "1",
|
||||
"label": "page test",
|
||||
},
|
||||
allow_redirects=False,
|
||||
)
|
||||
assert created.status_code == 302
|
||||
assert created.headers["location"] == "/admin/gateway?tab=quota"
|
||||
|
||||
rules = admin.get(f"{BASE_URL}/admin/gateway/quota-rules").json()["rules"]
|
||||
rule = next(r for r in rules if r["owner_id"] == owner and r["app_reference"] == app_ref)
|
||||
uid = rule["uid"]
|
||||
assert rule["limit_usd"] == 2.5
|
||||
|
||||
edit_page = admin.get(f"{BASE_URL}/admin/gateway/quota-rules/{uid}/edit")
|
||||
assert edit_page.status_code == 200
|
||||
assert edit_page.json()["form"]["limit_usd"] == "2.5"
|
||||
|
||||
updated = admin.post(
|
||||
f"{BASE_URL}/admin/gateway/quota-rules/{uid}/edit",
|
||||
data={
|
||||
"owner_kind": "user",
|
||||
"owner_id": owner,
|
||||
"app_reference": app_ref,
|
||||
"limit_usd": "5.0",
|
||||
"is_active": "1",
|
||||
"label": "page test updated",
|
||||
},
|
||||
allow_redirects=False,
|
||||
)
|
||||
assert updated.status_code == 302
|
||||
|
||||
rules = admin.get(f"{BASE_URL}/admin/gateway/quota-rules").json()["rules"]
|
||||
rule = next(r for r in rules if r["uid"] == uid)
|
||||
assert rule["limit_usd"] == 5.0
|
||||
assert rule["label"] == "page test updated"
|
||||
|
||||
_burn(owner, app_ref, 1.0)
|
||||
assert _spent(owner, app_ref) == 1.0
|
||||
reset_response = admin.post(
|
||||
f"{BASE_URL}/admin/gateway/quota-rules/{uid}/reset", allow_redirects=False
|
||||
)
|
||||
assert reset_response.status_code == 302
|
||||
assert _spent(owner, app_ref) == 0.0
|
||||
|
||||
deleted = admin.post(
|
||||
f"{BASE_URL}/admin/gateway/quota-rules/{uid}/delete", allow_redirects=False
|
||||
)
|
||||
assert deleted.status_code == 302
|
||||
assert (
|
||||
admin.get(f"{BASE_URL}/admin/gateway/quota-rules/{uid}/edit").status_code == 404
|
||||
)
|
||||
|
||||
|
||||
def test_quota_rule_page_requires_a_dimension(seeded_db):
|
||||
admin = admin_session(seeded_db)
|
||||
response = admin.post(
|
||||
f"{BASE_URL}/admin/gateway/quota-rules/new",
|
||||
data={"limit_usd": "1.0"},
|
||||
)
|
||||
assert response.status_code == 400
|
||||
assert "message" in response.json()["error"]
|
||||
|
||||
|
||||
def test_quota_rule_edit_page_404_for_missing_rule(seeded_db):
|
||||
admin = admin_session(seeded_db)
|
||||
from devplacepy.utils import generate_uid
|
||||
|
||||
missing = generate_uid()
|
||||
assert (
|
||||
admin.get(f"{BASE_URL}/admin/gateway/quota-rules/{missing}/edit").status_code
|
||||
== 404
|
||||
)
|
||||
assert (
|
||||
admin.post(f"{BASE_URL}/admin/gateway/quota-rules/{missing}/delete").status_code
|
||||
== 404
|
||||
)
|
||||
assert (
|
||||
admin.post(f"{BASE_URL}/admin/gateway/quota-rules/{missing}/reset").status_code
|
||||
== 404
|
||||
)
|
||||
|
||||
@@ -0,0 +1,82 @@
|
||||
# retoor <retoor@molodetz.nl>
|
||||
|
||||
import requests
|
||||
|
||||
from tests.api.admin.gateway.index import JSON_gateway, admin_session, member_key
|
||||
from tests.conftest import BASE_URL
|
||||
|
||||
|
||||
def test_stats_tab_renders_for_admin(seeded_db):
|
||||
admin = admin_session(seeded_db)
|
||||
response = admin.get(
|
||||
f"{BASE_URL}/admin/gateway?tab=stats", headers={"Accept": "text/html"}
|
||||
)
|
||||
assert response.status_code == 200
|
||||
assert "Model pool" in response.text
|
||||
assert "Model routes" not in response.text
|
||||
|
||||
|
||||
def test_stats_data_requires_admin(seeded_db):
|
||||
assert (
|
||||
requests.get(
|
||||
f"{BASE_URL}/admin/gateway/stats/data", headers=JSON_gateway, allow_redirects=False
|
||||
).status_code
|
||||
== 401
|
||||
)
|
||||
key = member_key()
|
||||
assert (
|
||||
requests.get(
|
||||
f"{BASE_URL}/admin/gateway/stats/data",
|
||||
headers={**JSON_gateway, "X-API-KEY": key},
|
||||
allow_redirects=False,
|
||||
).status_code
|
||||
== 403
|
||||
)
|
||||
|
||||
|
||||
def test_stats_data_shape_for_admin(seeded_db):
|
||||
admin = admin_session(seeded_db)
|
||||
response = admin.get(f"{BASE_URL}/admin/gateway/stats/data?range=24h", headers=JSON_gateway)
|
||||
assert response.status_code == 200
|
||||
data = response.json()
|
||||
for key in (
|
||||
"totals",
|
||||
"per_model",
|
||||
"per_endpoint",
|
||||
"per_provider",
|
||||
"timeseries",
|
||||
"status_codes",
|
||||
"streaming_split",
|
||||
"failure_reasons",
|
||||
"hourly_distribution",
|
||||
"recent_failures",
|
||||
"latency_histogram",
|
||||
"tokens_per_second_histogram",
|
||||
"bucket_seconds",
|
||||
):
|
||||
assert key in data
|
||||
|
||||
|
||||
def test_stats_data_rejects_unknown_range(seeded_db):
|
||||
admin = admin_session(seeded_db)
|
||||
response = admin.get(f"{BASE_URL}/admin/gateway/stats/data?range=nonsense", headers=JSON_gateway)
|
||||
assert response.status_code == 400
|
||||
|
||||
|
||||
def test_stats_models_returns_a_list(seeded_db):
|
||||
admin = admin_session(seeded_db)
|
||||
response = admin.get(f"{BASE_URL}/admin/gateway/stats/models", headers=JSON_gateway)
|
||||
assert response.status_code == 200
|
||||
assert isinstance(response.json()["models"], list)
|
||||
|
||||
|
||||
def test_stats_model_detail_for_an_unknown_model(seeded_db):
|
||||
admin = admin_session(seeded_db)
|
||||
response = admin.get(
|
||||
f"{BASE_URL}/admin/gateway/stats/model/default/never-called-model?range=24h",
|
||||
headers=JSON_gateway,
|
||||
)
|
||||
assert response.status_code == 200
|
||||
data = response.json()
|
||||
assert data["summary"]["total_requests"] == 0
|
||||
assert data["health"] is None
|
||||
@@ -0,0 +1,115 @@
|
||||
# retoor <retoor@molodetz.nl>
|
||||
|
||||
import time
|
||||
import pytest
|
||||
import requests
|
||||
from tests.conftest import BASE_URL
|
||||
from devplacepy.database import clear_settings_cache, get_table, refresh_snapshot
|
||||
from devplacepy.services.devii import tool_prefs
|
||||
|
||||
JSON = {"Accept": "application/json"}
|
||||
_counter = [0]
|
||||
|
||||
|
||||
@pytest.fixture(scope="module", autouse=True)
|
||||
def _settings(app_server):
|
||||
from devplacepy.database import set_setting
|
||||
|
||||
set_setting("rate_limit_per_minute", "1000000")
|
||||
set_setting("registration_open", "1")
|
||||
yield
|
||||
|
||||
|
||||
def _admin(seeded_db):
|
||||
refresh_snapshot()
|
||||
key = get_table("users").find_one(username="alice_test")["api_key"]
|
||||
s = requests.Session()
|
||||
s.headers.update({"X-API-KEY": key, **JSON})
|
||||
return s
|
||||
|
||||
|
||||
def _member():
|
||||
_counter[0] += 1
|
||||
name = f"dtmem{int(time.time() * 1000)}{_counter[0]}"
|
||||
requests.post(
|
||||
f"{BASE_URL}/auth/signup",
|
||||
data={
|
||||
"username": name,
|
||||
"email": f"{name}@t.dev",
|
||||
"password": "secret123",
|
||||
"confirm_password": "secret123",
|
||||
"birth_date": "1990-01-01",
|
||||
"accept_terms": "1",
|
||||
},
|
||||
allow_redirects=True,
|
||||
)
|
||||
refresh_snapshot()
|
||||
s = requests.Session()
|
||||
s.headers.update(
|
||||
{"X-API-KEY": get_table("users").find_one(username=name)["api_key"], **JSON}
|
||||
)
|
||||
return s
|
||||
|
||||
|
||||
def _all_tool_names():
|
||||
return set(tool_prefs.GROUPS_BY_TOOL_NAME)
|
||||
|
||||
|
||||
def test_devii_service_page_has_tools_tab(seeded_db):
|
||||
admin = _admin(seeded_db)
|
||||
r = admin.get(f"{BASE_URL}/admin/services/devii")
|
||||
assert r.status_code == 200
|
||||
assert 'data-tab="tools"' in r.text
|
||||
assert "devii-tools-form" in r.text
|
||||
|
||||
|
||||
def test_other_service_page_has_no_tools_tab(seeded_db):
|
||||
admin = _admin(seeded_db)
|
||||
r = admin.get(f"{BASE_URL}/admin/services/news")
|
||||
assert r.status_code == 200
|
||||
assert 'data-tab="tools"' not in r.text
|
||||
|
||||
|
||||
def test_admin_can_disable_and_reenable_a_tool(seeded_db):
|
||||
admin = _admin(seeded_db)
|
||||
all_names = _all_tool_names()
|
||||
try:
|
||||
enabled = sorted(all_names - {"create_post"})
|
||||
r = admin.post(
|
||||
f"{BASE_URL}/admin/services/devii/tools",
|
||||
data=[("enabled", name) for name in enabled],
|
||||
)
|
||||
assert r.status_code == 200, r.text[:300]
|
||||
assert r.json()["ok"] is True
|
||||
clear_settings_cache()
|
||||
assert tool_prefs.disabled_tool_names() == frozenset({"create_post"})
|
||||
|
||||
r2 = admin.post(
|
||||
f"{BASE_URL}/admin/services/devii/tools",
|
||||
data=[("enabled", name) for name in all_names],
|
||||
)
|
||||
assert r2.status_code == 200
|
||||
clear_settings_cache()
|
||||
assert tool_prefs.disabled_tool_names() == frozenset()
|
||||
finally:
|
||||
clear_settings_cache()
|
||||
tool_prefs.set_disabled_tool_names(set())
|
||||
|
||||
|
||||
def test_member_cannot_save_tool_config(app_server):
|
||||
member = _member()
|
||||
r = member.post(
|
||||
f"{BASE_URL}/admin/services/devii/tools",
|
||||
data={"enabled": "create_post"},
|
||||
allow_redirects=False,
|
||||
)
|
||||
assert r.status_code in (302, 303, 403)
|
||||
|
||||
|
||||
def test_guest_cannot_save_tool_config(app_server):
|
||||
r = requests.post(
|
||||
f"{BASE_URL}/admin/services/devii/tools",
|
||||
data={"enabled": "create_post"},
|
||||
allow_redirects=False,
|
||||
)
|
||||
assert r.status_code in (302, 303, 401)
|
||||
@@ -0,0 +1,138 @@
|
||||
# retoor <retoor@molodetz.nl>
|
||||
|
||||
import http.server
|
||||
import json
|
||||
import socket
|
||||
import socketserver
|
||||
import threading
|
||||
|
||||
import requests
|
||||
|
||||
from devplacepy.database import get_table, refresh_snapshot
|
||||
from tests.conftest import BASE_URL, login_user
|
||||
|
||||
|
||||
def _promote_to_admin(username: str) -> None:
|
||||
users = get_table("users")
|
||||
user = users.find_one(username=username)
|
||||
if user:
|
||||
users.update({"uid": user["uid"], "role": "Admin"}, ["uid"])
|
||||
|
||||
|
||||
def _admin_api_key(username: str) -> str:
|
||||
refresh_snapshot()
|
||||
return get_table("users").find_one(username=username)["api_key"]
|
||||
|
||||
|
||||
def _fake_models_upstream(model_ids):
|
||||
sock = socket.socket()
|
||||
sock.bind(("127.0.0.1", 0))
|
||||
port = sock.getsockname()[1]
|
||||
sock.close()
|
||||
body = json.dumps({"data": [{"id": m} for m in model_ids]}).encode()
|
||||
|
||||
class Handler(http.server.BaseHTTPRequestHandler):
|
||||
def do_GET(self):
|
||||
self.send_response(200)
|
||||
self.send_header("Content-Type", "application/json")
|
||||
self.end_headers()
|
||||
self.wfile.write(body)
|
||||
|
||||
def log_message(self, *args):
|
||||
pass
|
||||
|
||||
httpd = socketserver.TCPServer(("127.0.0.1", port), Handler)
|
||||
thread = threading.Thread(target=httpd.serve_forever, daemon=True)
|
||||
thread.start()
|
||||
return httpd, port
|
||||
|
||||
|
||||
def test_model_form_swaps_target_field_by_provider(page, seeded_db):
|
||||
user = seeded_db["alice"]
|
||||
_promote_to_admin(user["username"])
|
||||
key = _admin_api_key(user["username"])
|
||||
auth = {"X-API-KEY": key}
|
||||
|
||||
httpd, port = _fake_models_upstream(["vendor/known-a", "vendor/known-b"])
|
||||
listing_provider = f"e2elisting{port}"
|
||||
blind_provider = f"e2eblind{port}"
|
||||
try:
|
||||
requests.post(
|
||||
f"{BASE_URL}/admin/gateway/providers",
|
||||
json={
|
||||
"name": listing_provider,
|
||||
"base_url": f"http://127.0.0.1:{port}/v1/chat/completions",
|
||||
},
|
||||
headers=auth,
|
||||
)
|
||||
requests.post(
|
||||
f"{BASE_URL}/admin/gateway/providers",
|
||||
json={"name": blind_provider, "base_url": ""},
|
||||
headers=auth,
|
||||
)
|
||||
|
||||
login_user(page, user)
|
||||
page.goto(f"{BASE_URL}/admin/gateway/models/new", wait_until="domcontentloaded")
|
||||
|
||||
target_input = page.locator("#gw-model-target")
|
||||
target_select = page.locator("#gw-model-target-select")
|
||||
target_input.wait_for(state="visible")
|
||||
assert not target_select.is_visible()
|
||||
|
||||
page.select_option("#gw-model-provider", listing_provider)
|
||||
target_select.wait_for(state="visible", timeout=10000)
|
||||
assert not target_input.is_visible()
|
||||
assert target_select.get_attribute("required") is not None
|
||||
options = target_select.locator("option").all_inner_texts()
|
||||
assert "vendor/known-a" in options
|
||||
assert "vendor/known-b" in options
|
||||
|
||||
page.select_option("#gw-model-provider", blind_provider)
|
||||
target_input.wait_for(state="visible", timeout=10000)
|
||||
assert not target_select.is_visible()
|
||||
finally:
|
||||
httpd.shutdown()
|
||||
requests.delete(
|
||||
f"{BASE_URL}/admin/gateway/providers/{listing_provider}", headers=auth
|
||||
)
|
||||
requests.delete(
|
||||
f"{BASE_URL}/admin/gateway/providers/{blind_provider}", headers=auth
|
||||
)
|
||||
|
||||
|
||||
def test_model_form_submits_selected_model_from_dropdown(page, seeded_db):
|
||||
user = seeded_db["alice"]
|
||||
_promote_to_admin(user["username"])
|
||||
key = _admin_api_key(user["username"])
|
||||
auth = {"X-API-KEY": key}
|
||||
|
||||
httpd, port = _fake_models_upstream(["vendor/pick-me"])
|
||||
provider = f"e2esubmit{port}"
|
||||
source = f"e2esource{port}"
|
||||
try:
|
||||
requests.post(
|
||||
f"{BASE_URL}/admin/gateway/providers",
|
||||
json={
|
||||
"name": provider,
|
||||
"base_url": f"http://127.0.0.1:{port}/v1/chat/completions",
|
||||
},
|
||||
headers=auth,
|
||||
)
|
||||
|
||||
login_user(page, user)
|
||||
page.goto(f"{BASE_URL}/admin/gateway/models/new", wait_until="domcontentloaded")
|
||||
page.fill("#gw-model-source", source)
|
||||
page.select_option("#gw-model-provider", provider)
|
||||
page.locator("#gw-model-target-select").wait_for(state="visible", timeout=10000)
|
||||
page.select_option("#gw-model-target-select", "vendor/pick-me")
|
||||
page.click("button[type='submit']")
|
||||
page.wait_for_url(f"{BASE_URL}/admin/gateway?tab=models", wait_until="domcontentloaded")
|
||||
|
||||
listed = requests.get(f"{BASE_URL}/admin/gateway/models", headers=auth).json()
|
||||
row = next(m for m in listed["models"] if m["source_model"] == source)
|
||||
assert row["target_model"] == "vendor/pick-me"
|
||||
assert row["provider"] == provider
|
||||
finally:
|
||||
httpd.shutdown()
|
||||
requests.delete(f"{BASE_URL}/admin/gateway/models/{source}", headers=auth)
|
||||
requests.delete(f"{BASE_URL}/admin/gateway/providers/{provider}", headers=auth)
|
||||
@@ -101,3 +101,36 @@ def test_primary_admin_tool_denied_for_regular_admin_is_audited(monkeypatch):
|
||||
assert event_key == "security.authz.denied"
|
||||
assert kwargs["metadata"]["tool"] == "db_list_tables"
|
||||
assert kwargs["actor_role"] == "admin"
|
||||
|
||||
|
||||
def test_admin_disabled_tool_is_refused_even_for_admin(monkeypatch):
|
||||
from devplacepy.services.devii import tool_prefs
|
||||
|
||||
recorder = _patch_recorder(monkeypatch)
|
||||
monkeypatch.setattr(
|
||||
tool_prefs, "disabled_tool_names", lambda: frozenset({"create_post"})
|
||||
)
|
||||
dispatcher = _bare_dispatcher("user", "admin-uid-9", is_admin=True, is_primary_admin=True)
|
||||
|
||||
result = run_async(dispatcher.dispatch("create_post", {"content": "hello"}))
|
||||
|
||||
payload = json.loads(result)
|
||||
assert payload["error"] == "tool_disabled"
|
||||
assert len(recorder.calls) == 1
|
||||
event_key, kwargs = recorder.calls[0]
|
||||
assert event_key == "security.authz.denied"
|
||||
assert kwargs["metadata"]["tool"] == "create_post"
|
||||
assert kwargs["metadata"]["reason"] == "disabled by administrator"
|
||||
|
||||
|
||||
def test_non_disabled_tool_unaffected_by_disabled_set(monkeypatch):
|
||||
from devplacepy.services.devii import tool_prefs
|
||||
|
||||
monkeypatch.setattr(
|
||||
tool_prefs, "disabled_tool_names", lambda: frozenset({"create_post"})
|
||||
)
|
||||
dispatcher = _bare_dispatcher("user", "admin-uid-9", is_admin=True, is_primary_admin=True)
|
||||
|
||||
result = run_async(dispatcher.dispatch("admin_list_users", {}))
|
||||
|
||||
assert json.loads(result).get("error") != "tool_disabled"
|
||||
|
||||
@@ -0,0 +1,154 @@
|
||||
# retoor <retoor@molodetz.nl>
|
||||
|
||||
import json
|
||||
|
||||
from devplacepy.services.devii.agentic.compaction import (
|
||||
compact_messages,
|
||||
find_compaction_split,
|
||||
is_context_length_error,
|
||||
)
|
||||
from devplacepy.services.devii.errors import LLMError
|
||||
from tests.conftest import run_async
|
||||
|
||||
|
||||
def _error(status, body):
|
||||
return LLMError("Model endpoint returned error", status=status, body=body)
|
||||
|
||||
|
||||
def test_openrouter_style_message_detected():
|
||||
body = json.dumps(
|
||||
{
|
||||
"error": {
|
||||
"message": (
|
||||
"This endpoint's maximum context length is 131072 tokens. "
|
||||
"However, you requested about 403355 tokens (349784 of text "
|
||||
"input, 53571 of tool input). Please reduce the length of "
|
||||
"either one, or use the context-compression plugin."
|
||||
),
|
||||
"code": 400,
|
||||
"metadata": {"provider_name": None},
|
||||
}
|
||||
}
|
||||
)
|
||||
assert is_context_length_error(_error(400, body)) is True
|
||||
|
||||
|
||||
def test_openai_style_code_detected():
|
||||
body = json.dumps(
|
||||
{
|
||||
"error": {
|
||||
"message": "This model's maximum context length is 128000 tokens.",
|
||||
"type": "invalid_request_error",
|
||||
"param": None,
|
||||
"code": "context_length_exceeded",
|
||||
}
|
||||
}
|
||||
)
|
||||
assert is_context_length_error(_error(400, body)) is True
|
||||
|
||||
|
||||
def test_unrelated_400_not_detected():
|
||||
body = json.dumps({"error": {"message": "Invalid API key.", "code": 400}})
|
||||
assert is_context_length_error(_error(400, body)) is False
|
||||
|
||||
|
||||
def test_non_400_status_not_detected_even_with_matching_text():
|
||||
body = json.dumps(
|
||||
{"error": {"message": "maximum context length is 131072 tokens"}}
|
||||
)
|
||||
assert is_context_length_error(_error(429, body)) is False
|
||||
|
||||
|
||||
def test_malformed_body_falls_back_to_phrase_match():
|
||||
truncated = "maximum context length is 131072 tokens, please reduce the length"
|
||||
assert is_context_length_error(_error(400, truncated)) is True
|
||||
|
||||
|
||||
def test_malformed_body_with_no_match_is_false():
|
||||
assert is_context_length_error(_error(400, "not valid json at all")) is False
|
||||
|
||||
|
||||
class _StubLlm:
|
||||
def __init__(self, summary="a concise summary of the earlier turns"):
|
||||
self.summary = summary
|
||||
self.calls = 0
|
||||
|
||||
async def summarize(self, prompt):
|
||||
self.calls += 1
|
||||
return self.summary
|
||||
|
||||
|
||||
def _tool_call_message(name="run_tool"):
|
||||
return {
|
||||
"role": "assistant",
|
||||
"content": "",
|
||||
"tool_calls": [
|
||||
{"id": "c1", "function": {"name": name, "arguments": "{}"}}
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
def _tool_result_message(content="result"):
|
||||
return {"role": "tool", "tool_call_id": "c1", "name": "run_tool", "content": content}
|
||||
|
||||
|
||||
def _long_tool_heavy_conversation(rounds=20):
|
||||
messages = [
|
||||
{"role": "system", "content": "sys"},
|
||||
{"role": "user", "content": "start the long task"},
|
||||
]
|
||||
for i in range(rounds):
|
||||
messages.append(_tool_call_message())
|
||||
messages.append(_tool_result_message(f"result {i}" * 200))
|
||||
return messages
|
||||
|
||||
|
||||
def test_find_compaction_split_prefers_a_user_message():
|
||||
messages = [
|
||||
{"role": "system", "content": "sys"},
|
||||
{"role": "user", "content": "first"},
|
||||
{"role": "assistant", "content": "reply"},
|
||||
{"role": "user", "content": "second"},
|
||||
{"role": "assistant", "content": "reply2"},
|
||||
{"role": "user", "content": "third"},
|
||||
{"role": "assistant", "content": "reply3"},
|
||||
]
|
||||
split = find_compaction_split(messages, keep_tail=2)
|
||||
assert messages[split]["role"] == "user"
|
||||
|
||||
|
||||
def test_find_compaction_split_falls_back_to_a_non_tool_boundary_without_a_recent_user_message():
|
||||
messages = _long_tool_heavy_conversation(rounds=20)
|
||||
split = find_compaction_split(messages, keep_tail=4)
|
||||
assert split > 1
|
||||
assert messages[split].get("role") != "tool"
|
||||
|
||||
|
||||
def test_find_compaction_split_never_lands_inside_a_tool_result_run():
|
||||
messages = _long_tool_heavy_conversation(rounds=30)
|
||||
for keep_tail in (2, 3, 4, 5, 8, 10, 15):
|
||||
split = find_compaction_split(messages, keep_tail)
|
||||
assert messages[split].get("role") != "tool", (
|
||||
f"keep_tail={keep_tail} split at a tool message, orphaning its tool_calls"
|
||||
)
|
||||
|
||||
|
||||
def test_compact_messages_shrinks_a_tool_heavy_conversation_with_no_recent_user_message():
|
||||
messages = _long_tool_heavy_conversation(rounds=20)
|
||||
original_len = len(messages)
|
||||
llm = _StubLlm()
|
||||
compacted = run_async(compact_messages(llm, messages, keep_tail=4))
|
||||
assert llm.calls == 1
|
||||
assert len(compacted) < original_len
|
||||
assert compacted[0]["role"] == "system"
|
||||
assert "[compacted earlier turns]" in compacted[1]["content"]
|
||||
assert compacted[-1] == messages[-1]
|
||||
|
||||
|
||||
def test_compact_messages_tail_never_starts_with_a_dangling_tool_result():
|
||||
messages = _long_tool_heavy_conversation(rounds=25)
|
||||
llm = _StubLlm()
|
||||
compacted = run_async(compact_messages(llm, messages, keep_tail=6))
|
||||
tail = compacted[2:]
|
||||
assert tail
|
||||
assert tail[0].get("role") != "tool"
|
||||
@@ -1,7 +1,17 @@
|
||||
# retoor <retoor@molodetz.nl>
|
||||
|
||||
import dataclasses
|
||||
import json
|
||||
from devplacepy.services.devii.agentic.loop import _run_tool_call
|
||||
|
||||
from devplacepy.services.devii.agentic.compaction import context_size
|
||||
from devplacepy.services.devii.agentic.loop import (
|
||||
MAX_CONTEXT_OVERFLOW_RETRIES,
|
||||
_run_tool_call,
|
||||
react_loop,
|
||||
)
|
||||
from devplacepy.services.devii.agentic.state import AgentState
|
||||
from devplacepy.services.devii.config import load_settings
|
||||
from devplacepy.services.devii.errors import LLMError
|
||||
from tests.conftest import run_async
|
||||
class _FakeDispatcher:
|
||||
def __init__(self):
|
||||
@@ -54,3 +64,173 @@ def test_missing_arguments_defaults_to_empty_object():
|
||||
out = _run({"function": {"name": "auth_status"}}, dispatcher)
|
||||
assert out["status"] == "ok"
|
||||
assert dispatcher.calls == [("auth_status", {})]
|
||||
|
||||
|
||||
_CONTEXT_LENGTH_BODY = json.dumps(
|
||||
{
|
||||
"error": {
|
||||
"message": (
|
||||
"This endpoint's maximum context length is 131072 tokens. "
|
||||
"However, you requested about 403355 tokens. Please reduce "
|
||||
"the length."
|
||||
),
|
||||
"code": 400,
|
||||
}
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def _context_length_error():
|
||||
return LLMError(
|
||||
"Model endpoint returned 400: over limit", status=400, body=_CONTEXT_LENGTH_BODY
|
||||
)
|
||||
|
||||
|
||||
def _settings_for_test(keep_tail=4, threshold=10**9):
|
||||
return dataclasses.replace(
|
||||
load_settings(),
|
||||
context_compact_threshold=threshold,
|
||||
context_keep_tail=keep_tail,
|
||||
)
|
||||
|
||||
|
||||
def _long_message_history(count=10):
|
||||
messages = [{"role": "system", "content": "system prompt"}]
|
||||
for i in range(count):
|
||||
role = "user" if i % 2 == 0 else "assistant"
|
||||
messages.append({"role": role, "content": f"turn {i}"})
|
||||
return messages
|
||||
|
||||
|
||||
class _FakeLLM:
|
||||
def __init__(self, complete_results):
|
||||
self._complete_results = list(complete_results)
|
||||
self.complete_calls = 0
|
||||
self.summarize_calls = 0
|
||||
|
||||
async def complete(self, messages, tools):
|
||||
self.complete_calls += 1
|
||||
result = self._complete_results[
|
||||
min(self.complete_calls, len(self._complete_results)) - 1
|
||||
]
|
||||
if isinstance(result, Exception):
|
||||
raise result
|
||||
return result
|
||||
|
||||
async def summarize(self, text):
|
||||
self.summarize_calls += 1
|
||||
return "compacted summary"
|
||||
|
||||
|
||||
def test_context_overflow_triggers_compaction_and_retries():
|
||||
llm = _FakeLLM(
|
||||
[_context_length_error(), {"role": "assistant", "content": "Recovered answer"}]
|
||||
)
|
||||
messages = _long_message_history()
|
||||
result = run_async(
|
||||
react_loop(
|
||||
llm,
|
||||
_FakeDispatcher(),
|
||||
messages,
|
||||
tools=[],
|
||||
state=AgentState(),
|
||||
settings=_settings_for_test(),
|
||||
max_iterations=5,
|
||||
plan_required=False,
|
||||
verify_required=False,
|
||||
)
|
||||
)
|
||||
assert result == "Recovered answer"
|
||||
assert llm.complete_calls == 2
|
||||
assert llm.summarize_calls == 1
|
||||
assert not result.startswith("[model error]")
|
||||
|
||||
|
||||
def test_context_overflow_gives_up_after_max_retries_with_clear_message():
|
||||
llm = _FakeLLM([_context_length_error()])
|
||||
messages = _long_message_history()
|
||||
result = run_async(
|
||||
react_loop(
|
||||
llm,
|
||||
_FakeDispatcher(),
|
||||
messages,
|
||||
tools=[],
|
||||
state=AgentState(),
|
||||
settings=_settings_for_test(),
|
||||
max_iterations=10,
|
||||
plan_required=False,
|
||||
verify_required=False,
|
||||
)
|
||||
)
|
||||
assert result.startswith("[model error]")
|
||||
assert "context length" in result.lower() or "400" in result
|
||||
assert llm.complete_calls == MAX_CONTEXT_OVERFLOW_RETRIES + 1
|
||||
assert 0 < llm.summarize_calls <= MAX_CONTEXT_OVERFLOW_RETRIES
|
||||
|
||||
|
||||
def test_non_context_length_error_never_triggers_compaction():
|
||||
llm = _FakeLLM([LLMError("Model endpoint returned 500: boom", status=500, body="{}")])
|
||||
messages = _long_message_history()
|
||||
result = run_async(
|
||||
react_loop(
|
||||
llm,
|
||||
_FakeDispatcher(),
|
||||
messages,
|
||||
tools=[],
|
||||
state=AgentState(),
|
||||
settings=_settings_for_test(),
|
||||
max_iterations=5,
|
||||
plan_required=False,
|
||||
verify_required=False,
|
||||
)
|
||||
)
|
||||
assert result == "[model error] Model endpoint returned 500: boom"
|
||||
assert llm.complete_calls == 1
|
||||
assert llm.summarize_calls == 0
|
||||
|
||||
|
||||
class _FakeLLMWithRealLimit:
|
||||
def __init__(self, simulated_limit_chars):
|
||||
self.simulated_limit_chars = simulated_limit_chars
|
||||
self.complete_calls = 0
|
||||
self.summarize_calls = 0
|
||||
|
||||
async def complete(self, messages, tools):
|
||||
self.complete_calls += 1
|
||||
if context_size(messages) > self.simulated_limit_chars:
|
||||
raise _context_length_error()
|
||||
return {"role": "assistant", "content": "Recovered answer"}
|
||||
|
||||
async def summarize(self, text):
|
||||
self.summarize_calls += 1
|
||||
return "short summary"
|
||||
|
||||
|
||||
def test_one_oversized_tail_message_alone_still_recovers():
|
||||
giant = "X" * 300_000
|
||||
messages = _long_message_history(16)
|
||||
messages.append(
|
||||
{"role": "tool", "tool_call_id": "1", "name": "big_tool", "content": giant}
|
||||
)
|
||||
messages.append({"role": "user", "content": "please continue"})
|
||||
|
||||
llm = _FakeLLMWithRealLimit(simulated_limit_chars=30_000)
|
||||
result = run_async(
|
||||
react_loop(
|
||||
llm,
|
||||
_FakeDispatcher(),
|
||||
messages,
|
||||
tools=[],
|
||||
state=AgentState(),
|
||||
settings=_settings_for_test(keep_tail=4),
|
||||
max_iterations=10,
|
||||
plan_required=False,
|
||||
verify_required=False,
|
||||
)
|
||||
)
|
||||
|
||||
assert result == "Recovered answer"
|
||||
assert llm.complete_calls > MAX_CONTEXT_OVERFLOW_RETRIES - 1
|
||||
giant_message = next(m for m in messages if m.get("name") == "big_tool")
|
||||
assert len(giant_message["content"]) < len(giant)
|
||||
assert "truncated" in giant_message["content"]
|
||||
|
||||
@@ -0,0 +1,56 @@
|
||||
# retoor <retoor@molodetz.nl>
|
||||
|
||||
from devplacepy.services.devii import tool_prefs
|
||||
|
||||
|
||||
def test_disabled_tool_names_round_trips(local_db):
|
||||
try:
|
||||
tool_prefs.set_disabled_tool_names({"create_post", "delete_post"})
|
||||
assert tool_prefs.disabled_tool_names() == frozenset({"create_post", "delete_post"})
|
||||
finally:
|
||||
tool_prefs.set_disabled_tool_names(set())
|
||||
|
||||
|
||||
def test_set_disabled_tool_names_drops_unknown_names(local_db):
|
||||
try:
|
||||
tool_prefs.set_disabled_tool_names({"create_post", "not_a_real_tool_xyz"})
|
||||
assert tool_prefs.disabled_tool_names() == frozenset({"create_post"})
|
||||
finally:
|
||||
tool_prefs.set_disabled_tool_names(set())
|
||||
|
||||
|
||||
def test_disabled_tool_names_empty_by_default(local_db):
|
||||
tool_prefs.set_disabled_tool_names(set())
|
||||
assert tool_prefs.disabled_tool_names() == frozenset()
|
||||
|
||||
|
||||
def test_filter_disabled_removes_matching_schemas():
|
||||
schemas = [
|
||||
{"function": {"name": "create_post"}},
|
||||
{"function": {"name": "list_posts"}},
|
||||
]
|
||||
result = tool_prefs.filter_disabled(schemas, disabled=frozenset({"create_post"}))
|
||||
assert [s["function"]["name"] for s in result] == ["list_posts"]
|
||||
|
||||
|
||||
def test_filter_disabled_is_a_no_op_when_nothing_disabled():
|
||||
schemas = [{"function": {"name": "create_post"}}]
|
||||
assert tool_prefs.filter_disabled(schemas, disabled=frozenset()) == schemas
|
||||
|
||||
|
||||
def test_group_overview_covers_every_group_and_marks_disabled(local_db):
|
||||
try:
|
||||
tool_prefs.set_disabled_tool_names({"create_post"})
|
||||
overview = tool_prefs.group_overview()
|
||||
assert len(overview) == len(tool_prefs.GROUPS)
|
||||
posts_group = next(g for g in overview if g["key"] == "posts")
|
||||
create_post_tool = next(t for t in posts_group["tools"] if t["name"] == "create_post")
|
||||
assert create_post_tool["disabled"] is True
|
||||
assert posts_group["enabled_count"] == posts_group["total_count"] - 1
|
||||
finally:
|
||||
tool_prefs.set_disabled_tool_names(set())
|
||||
|
||||
|
||||
def test_groups_by_tool_name_covers_every_action_in_every_group():
|
||||
total_actions = sum(len(actions) for actions in tool_prefs.GROUPS.values())
|
||||
assert len(tool_prefs.GROUPS_BY_TOOL_NAME) == total_actions
|
||||
@@ -41,6 +41,10 @@ class FakeResp_openai_gateway:
|
||||
content = payload["choices"][0]["message"].get("content") or ""
|
||||
except (KeyError, IndexError, TypeError):
|
||||
content = ""
|
||||
try:
|
||||
reasoning = payload["choices"][0]["message"].get("reasoning") or ""
|
||||
except (KeyError, IndexError, TypeError):
|
||||
reasoning = ""
|
||||
|
||||
def frame(delta=None, finish=None, usage=None):
|
||||
body = {"id": chunk_id, "object": "chat.completion.chunk", "model": model}
|
||||
@@ -52,6 +56,8 @@ class FakeResp_openai_gateway:
|
||||
return f"data: {json.dumps(body)}"
|
||||
|
||||
yield frame({"role": "assistant"})
|
||||
for i in range(0, len(reasoning), 5):
|
||||
yield frame({"reasoning": reasoning[i : i + 5]})
|
||||
for i in range(0, len(content), 5):
|
||||
yield frame({"content": content[i : i + 5]})
|
||||
yield frame({}, finish="stop")
|
||||
@@ -693,6 +699,59 @@ class FakeAlwaysFailClient_openai_gateway:
|
||||
pass
|
||||
|
||||
|
||||
class FakeBadRequestClient_openai_gateway:
|
||||
def __init__(self, *a, **k):
|
||||
self.calls = []
|
||||
|
||||
def build_request(self, method, url, headers=None, json=None, content=None):
|
||||
return FakeRequest(method, url, json)
|
||||
|
||||
async def send(self, request, stream=False):
|
||||
self.calls.append((request.url, request.json_body))
|
||||
return FakeResp_openai_gateway(
|
||||
status=400, payload={"error": {"message": "invalid request: bad param"}}
|
||||
)
|
||||
|
||||
async def aclose(self):
|
||||
pass
|
||||
|
||||
|
||||
def test_chat_does_not_fall_back_on_an_unrecoverable_bad_request(local_db, monkeypatch):
|
||||
from devplacepy.services.openai_gateway import routing
|
||||
|
||||
monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeBadRequestClient_openai_gateway)
|
||||
routing.model_store.set(
|
||||
routing.ModelRouteIn(source_model="fb-badreq-backup", target_model="backup-target")
|
||||
)
|
||||
routing.model_store.set(
|
||||
routing.ModelRouteIn(
|
||||
source_model="fb-badreq-primary",
|
||||
target_model="primary-target",
|
||||
fallback_model="fb-badreq-backup",
|
||||
)
|
||||
)
|
||||
try:
|
||||
svc = GatewayService()
|
||||
cfg = svc.effective_config()
|
||||
cfg["gateway_vision_enabled"] = False
|
||||
cfg["gateway_max_retries"] = 0
|
||||
rt = svc.runtime()
|
||||
response = run_async(
|
||||
rt.handle_chat(
|
||||
{"model": "fb-badreq-primary", "messages": [{"role": "user", "content": "hi"}]},
|
||||
cfg,
|
||||
("guest", "no_fallback_on_bad_request"),
|
||||
"test",
|
||||
"default",
|
||||
)
|
||||
)
|
||||
assert response.status_code == 400
|
||||
assert len(rt._client.calls) == 1
|
||||
finally:
|
||||
routing.model_store.remove("fb-badreq-primary")
|
||||
routing.model_store.remove("fb-badreq-backup")
|
||||
|
||||
|
||||
def test_chat_falls_back_when_the_primary_model_fails(local_db, monkeypatch):
|
||||
from devplacepy.services.openai_gateway import routing
|
||||
|
||||
@@ -735,6 +794,127 @@ def test_chat_falls_back_when_the_primary_model_fails(local_db, monkeypatch):
|
||||
routing.model_store.remove("fb-backup-route")
|
||||
|
||||
|
||||
class FakeContextLengthClient_openai_gateway:
|
||||
def __init__(self, *a, **k):
|
||||
self.calls = []
|
||||
|
||||
def build_request(self, method, url, headers=None, json=None, content=None):
|
||||
return FakeRequest(method, url, json)
|
||||
|
||||
async def send(self, request, stream=False):
|
||||
self.calls.append((request.url, request.json_body))
|
||||
body = request.json_body or {}
|
||||
model = body.get("model")
|
||||
if model == "ctx-primary-target":
|
||||
return FakeResp_openai_gateway(
|
||||
status=400,
|
||||
payload={"error": {"message": "maximum context length exceeded"}},
|
||||
)
|
||||
return FakeResp_openai_gateway(
|
||||
payload={
|
||||
"id": "x",
|
||||
"model": model,
|
||||
"choices": [{"message": {"content": "fallback ok"}}],
|
||||
}
|
||||
)
|
||||
|
||||
async def aclose(self):
|
||||
pass
|
||||
|
||||
|
||||
def test_chat_falls_back_on_a_context_length_error(local_db, monkeypatch):
|
||||
from devplacepy.services.openai_gateway import routing
|
||||
|
||||
monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeContextLengthClient_openai_gateway)
|
||||
routing.model_store.set(
|
||||
routing.ModelRouteIn(source_model="ctx-backup-route", target_model="ctx-backup-target")
|
||||
)
|
||||
routing.model_store.set(
|
||||
routing.ModelRouteIn(
|
||||
source_model="ctx-primary-route",
|
||||
target_model="ctx-primary-target",
|
||||
fallback_model="ctx-backup-route",
|
||||
)
|
||||
)
|
||||
try:
|
||||
svc = GatewayService()
|
||||
cfg = svc.effective_config()
|
||||
cfg["gateway_vision_enabled"] = False
|
||||
cfg["gateway_max_retries"] = 0
|
||||
rt = svc.runtime()
|
||||
response = run_async(
|
||||
rt.handle_chat(
|
||||
{"model": "ctx-primary-route", "messages": [{"role": "user", "content": "hi"}]},
|
||||
cfg,
|
||||
("guest", "context_length_fallback_success"),
|
||||
"test",
|
||||
"default",
|
||||
)
|
||||
)
|
||||
assert response.status_code == 200
|
||||
assert rt._client.calls[0][1]["model"] == "ctx-primary-target"
|
||||
assert rt._client.calls[-1][1]["model"] == "ctx-backup-target"
|
||||
row = get_table("gateway_usage_ledger").find_one(
|
||||
owner_id="context_length_fallback_success"
|
||||
)
|
||||
assert row["success"] == 1
|
||||
assert row["fallback_used_route"] == "ctx-backup-route"
|
||||
finally:
|
||||
routing.model_store.remove("ctx-primary-route")
|
||||
routing.model_store.remove("ctx-backup-route")
|
||||
|
||||
|
||||
def test_chat_skips_a_too_small_primary_and_goes_straight_to_fallback(local_db, monkeypatch):
|
||||
from devplacepy.services.openai_gateway import routing
|
||||
|
||||
monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeContextLengthClient_openai_gateway)
|
||||
routing.model_store.set(
|
||||
routing.ModelRouteIn(
|
||||
source_model="ctx-precheck-backup",
|
||||
target_model="ctx-backup-target",
|
||||
context_window=1_000_000,
|
||||
)
|
||||
)
|
||||
routing.model_store.set(
|
||||
routing.ModelRouteIn(
|
||||
source_model="ctx-precheck-primary",
|
||||
target_model="ctx-primary-target",
|
||||
context_window=50,
|
||||
fallback_model="ctx-precheck-backup",
|
||||
)
|
||||
)
|
||||
try:
|
||||
svc = GatewayService()
|
||||
cfg = svc.effective_config()
|
||||
cfg["gateway_vision_enabled"] = False
|
||||
cfg["gateway_max_retries"] = 0
|
||||
rt = svc.runtime()
|
||||
long_message = "word " * 2000
|
||||
response = run_async(
|
||||
rt.handle_chat(
|
||||
{
|
||||
"model": "ctx-precheck-primary",
|
||||
"messages": [{"role": "user", "content": long_message}],
|
||||
},
|
||||
cfg,
|
||||
("guest", "context_precheck_skips_primary"),
|
||||
"test",
|
||||
"default",
|
||||
)
|
||||
)
|
||||
assert response.status_code == 200
|
||||
# Only the fallback was ever called - the primary was known too small.
|
||||
assert len(rt._client.calls) == 1
|
||||
assert rt._client.calls[0][1]["model"] == "ctx-backup-target"
|
||||
row = get_table("gateway_usage_ledger").find_one(
|
||||
owner_id="context_precheck_skips_primary"
|
||||
)
|
||||
assert row["fallback_used_route"] == "ctx-precheck-backup"
|
||||
finally:
|
||||
routing.model_store.remove("ctx-precheck-primary")
|
||||
routing.model_store.remove("ctx-precheck-backup")
|
||||
|
||||
|
||||
def test_chat_falls_back_via_molodetz_when_the_client_sends_an_unrouted_model_name(
|
||||
local_db, monkeypatch
|
||||
):
|
||||
@@ -1271,6 +1451,50 @@ def test_stream_records_ttft_and_inter_token_in_ledger(local_db, monkeypatch):
|
||||
assert row["inter_token_ms"] is not None and row["inter_token_ms"] >= 0.0
|
||||
|
||||
|
||||
class FakeClientReasoning_openai_gateway(FakeClient_openai_gateway):
|
||||
async def send(self, request, stream=False):
|
||||
self.calls.append((request.url, request.json_body))
|
||||
body = request.json_body or {}
|
||||
return FakeResp_openai_gateway(
|
||||
payload={
|
||||
"id": "x",
|
||||
"model": body.get("model"),
|
||||
"choices": [{"message": {"content": "x", "reasoning": "aaaaa"}}],
|
||||
"usage": {"prompt_tokens": 7, "completion_tokens": 3, "total_tokens": 10},
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def test_ollama_reasoning_delta_is_counted_as_a_content_chunk(local_db, monkeypatch):
|
||||
monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeClientReasoning_openai_gateway)
|
||||
svc = GatewayService()
|
||||
cfg = svc.effective_config()
|
||||
rt = svc.runtime()
|
||||
resp = run_async(
|
||||
rt.handle_chat(
|
||||
{
|
||||
"messages": [{"role": "user", "content": "hi"}],
|
||||
"stream": True,
|
||||
"stream_options": {"include_usage": True},
|
||||
},
|
||||
cfg,
|
||||
("guest", "ollama_reasoning_probe"),
|
||||
"test",
|
||||
"default",
|
||||
)
|
||||
)
|
||||
|
||||
async def drain():
|
||||
async for _ in resp.body_iterator:
|
||||
pass
|
||||
|
||||
run_async(drain())
|
||||
|
||||
row = get_table("gateway_usage_ledger").find_one(owner_id="ollama_reasoning_probe")
|
||||
assert row is not None
|
||||
assert row["inter_token_ms"] is not None and row["inter_token_ms"] >= 0.0
|
||||
|
||||
|
||||
def test_stream_client_disconnect_records_failure_and_closes_upstream(local_db, monkeypatch):
|
||||
monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeClient_openai_gateway)
|
||||
svc = GatewayService()
|
||||
@@ -1552,6 +1776,153 @@ def test_no_upstream_model_header_keeps_our_own_model(local_db, monkeypatch):
|
||||
assert resp.headers["X-Gateway-Model"] == "deepseek-chat"
|
||||
|
||||
|
||||
class FakeEmbeddedErrorClient_openai_gateway:
|
||||
"""Mimics OpenRouter's documented behavior of answering 200 OK with the
|
||||
failure embedded in the JSON body (openrouter.ai/docs/api-reference/errors)
|
||||
instead of a non-2xx status."""
|
||||
|
||||
def __init__(self, *a, **k):
|
||||
self.calls = []
|
||||
|
||||
def build_request(self, method, url, headers=None, json=None, content=None):
|
||||
return FakeRequest(method, url, json)
|
||||
|
||||
async def send(self, request, stream=False):
|
||||
self.calls.append((request.url, request.json_body))
|
||||
body = request.json_body or {}
|
||||
model = body.get("model")
|
||||
if model == "primary-target":
|
||||
return FakeResp_openai_gateway(
|
||||
status=200,
|
||||
payload={"error": {"message": "no provider available", "code": 502}},
|
||||
)
|
||||
return FakeResp_openai_gateway(
|
||||
payload={
|
||||
"id": "x",
|
||||
"model": model,
|
||||
"choices": [{"message": {"content": "fallback ok"}}],
|
||||
}
|
||||
)
|
||||
|
||||
async def aclose(self):
|
||||
pass
|
||||
|
||||
|
||||
def test_chat_falls_back_when_upstream_returns_200_with_an_embedded_error(
|
||||
local_db, monkeypatch
|
||||
):
|
||||
from devplacepy.services.openai_gateway import routing
|
||||
|
||||
monkeypatch.setattr(
|
||||
gwmod.httpx, "AsyncClient", FakeEmbeddedErrorClient_openai_gateway
|
||||
)
|
||||
routing.model_store.set(
|
||||
routing.ModelRouteIn(source_model="eb-backup-route", target_model="backup-target")
|
||||
)
|
||||
routing.model_store.set(
|
||||
routing.ModelRouteIn(
|
||||
source_model="eb-primary-route",
|
||||
target_model="primary-target",
|
||||
fallback_model="eb-backup-route",
|
||||
)
|
||||
)
|
||||
try:
|
||||
svc = GatewayService()
|
||||
cfg = svc.effective_config()
|
||||
cfg["gateway_vision_enabled"] = False
|
||||
cfg["gateway_max_retries"] = 0
|
||||
rt = svc.runtime()
|
||||
response = run_async(
|
||||
rt.handle_chat(
|
||||
{"model": "eb-primary-route", "messages": [{"role": "user", "content": "hi"}]},
|
||||
cfg,
|
||||
("guest", "embedded_error_chat"),
|
||||
"test",
|
||||
"default",
|
||||
)
|
||||
)
|
||||
assert response.status_code == 200
|
||||
assert rt._client.calls[0][1]["model"] == "primary-target"
|
||||
assert rt._client.calls[-1][1]["model"] == "backup-target"
|
||||
row = get_table("gateway_usage_ledger").find_one(owner_id="embedded_error_chat")
|
||||
assert row is not None
|
||||
assert row["requested_model"] == "eb-primary-route"
|
||||
assert row["model"] == "backup-target"
|
||||
assert row["success"] == 1
|
||||
finally:
|
||||
routing.model_store.remove("eb-primary-route")
|
||||
routing.model_store.remove("eb-backup-route")
|
||||
|
||||
|
||||
class FakeAlwaysEmbeddedErrorClient_openai_gateway:
|
||||
def __init__(self, *a, **k):
|
||||
self.calls = []
|
||||
|
||||
def build_request(self, method, url, headers=None, json=None, content=None):
|
||||
return FakeRequest(method, url, json)
|
||||
|
||||
async def send(self, request, stream=False):
|
||||
self.calls.append((request.url, request.json_body))
|
||||
return FakeResp_openai_gateway(
|
||||
status=200,
|
||||
payload={"error": {"message": "no provider available", "code": 502}},
|
||||
)
|
||||
|
||||
async def aclose(self):
|
||||
pass
|
||||
|
||||
|
||||
def test_chat_reports_502_when_upstream_returns_200_with_an_embedded_error_and_no_fallback(
|
||||
local_db, monkeypatch
|
||||
):
|
||||
monkeypatch.setattr(
|
||||
gwmod.httpx, "AsyncClient", FakeAlwaysEmbeddedErrorClient_openai_gateway
|
||||
)
|
||||
svc = GatewayService()
|
||||
cfg = svc.effective_config()
|
||||
cfg["gateway_vision_enabled"] = False
|
||||
cfg["gateway_max_retries"] = 0
|
||||
rt = svc.runtime()
|
||||
response = run_async(
|
||||
rt.handle_chat(
|
||||
{"messages": [{"role": "user", "content": "hi"}]},
|
||||
cfg,
|
||||
("guest", "embedded_error_no_fallback"),
|
||||
"test",
|
||||
"default",
|
||||
)
|
||||
)
|
||||
assert response.status_code == 502
|
||||
row = get_table("gateway_usage_ledger").find_one(owner_id="embedded_error_no_fallback")
|
||||
assert row is not None
|
||||
assert row["success"] == 0
|
||||
assert row["error_category"] == "upstream_error"
|
||||
|
||||
|
||||
def test_ollama_dialect_sends_reasoning_effort_alongside_think(local_db, monkeypatch):
|
||||
# Ollama's native /api/chat honors a boolean `think`, but its
|
||||
# OpenAI-compatible /v1/chat/completions layer (what this gateway
|
||||
# actually calls) does not - it maps reasoning_effort/reasoning
|
||||
# instead (github.com/ollama/ollama issues #15288, #15293, #14820).
|
||||
monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeClient_openai_gateway)
|
||||
svc = GatewayService()
|
||||
cfg = svc.effective_config()
|
||||
cfg["gateway_upstream_url"] = "http://127.0.0.1:11434/v1/chat/completions"
|
||||
rt = svc.runtime()
|
||||
run_async(
|
||||
rt.handle_chat(
|
||||
{"messages": [{"role": "user", "content": "hi"}]},
|
||||
cfg,
|
||||
("guest", "ollama_reasoning_effort"),
|
||||
"test",
|
||||
"default",
|
||||
)
|
||||
)
|
||||
body = rt._client.calls[-1][1]
|
||||
assert body["think"] is False
|
||||
assert body["reasoning_effort"] == "none"
|
||||
|
||||
|
||||
def test_hostile_upstream_model_header_is_ignored_end_to_end(local_db, monkeypatch):
|
||||
monkeypatch.setattr(
|
||||
gwmod.httpx, "AsyncClient", FakeClientHostileModelHeader_openai_gateway
|
||||
|
||||
@@ -0,0 +1,83 @@
|
||||
# retoor <retoor@molodetz.nl>
|
||||
|
||||
from devplacepy.services.openai_gateway import model_health
|
||||
|
||||
|
||||
def setup_function():
|
||||
model_health.reset()
|
||||
|
||||
|
||||
def test_speed_reward_is_neutral_without_data():
|
||||
assert model_health.speed_reward(None) == model_health.NEUTRAL_REWARD
|
||||
assert model_health.speed_reward(0) == model_health.NEUTRAL_REWARD
|
||||
|
||||
|
||||
def test_speed_reward_increases_with_throughput():
|
||||
slow = model_health.speed_reward(5)
|
||||
fast = model_health.speed_reward(50)
|
||||
assert 0 < slow < fast < 1
|
||||
|
||||
|
||||
def test_latency_reward_is_one_without_data():
|
||||
assert model_health.latency_reward(None) == 1.0
|
||||
assert model_health.latency_reward(0) == 1.0
|
||||
|
||||
|
||||
def test_latency_reward_decreases_with_latency():
|
||||
fast = model_health.latency_reward(500)
|
||||
slow = model_health.latency_reward(20000)
|
||||
assert 0 < slow < fast <= 1
|
||||
|
||||
|
||||
def test_untested_model_has_neutral_weight():
|
||||
health = model_health.ModelHealth()
|
||||
assert health.weight() == 0.5
|
||||
|
||||
|
||||
def test_record_outcome_success_improves_weight_over_failures():
|
||||
model_health.record_outcome("openrouter", "fast-model", True, latency_ms=200, tokens_per_second=80)
|
||||
model_health.record_outcome("openrouter", "slow-model", False)
|
||||
fast_weight = model_health.snapshot_for("openrouter", "fast-model")["weight"]
|
||||
slow_weight = model_health.snapshot_for("openrouter", "slow-model")["weight"]
|
||||
assert fast_weight > 0.5
|
||||
assert slow_weight < 0.5
|
||||
|
||||
|
||||
def test_repeated_failures_open_the_circuit_for_display_only():
|
||||
for _ in range(model_health.CIRCUIT_BREAKER_FAILURE_THRESHOLD):
|
||||
model_health.record_outcome("deepseek", "flaky-model", False)
|
||||
snapshot = model_health.snapshot_for("deepseek", "flaky-model")
|
||||
assert snapshot["circuit_open"] is True
|
||||
assert snapshot["consecutive_failures"] == model_health.CIRCUIT_BREAKER_FAILURE_THRESHOLD
|
||||
|
||||
|
||||
def test_success_resets_consecutive_failures_and_circuit():
|
||||
model_health.record_outcome("deepseek", "recovering-model", False)
|
||||
model_health.record_outcome("deepseek", "recovering-model", False)
|
||||
model_health.record_outcome("deepseek", "recovering-model", True, latency_ms=100, tokens_per_second=50)
|
||||
snapshot = model_health.snapshot_for("deepseek", "recovering-model")
|
||||
assert snapshot["consecutive_failures"] == 0
|
||||
assert snapshot["circuit_open"] is False
|
||||
|
||||
|
||||
def test_snapshot_for_unknown_model_is_none():
|
||||
assert model_health.snapshot_for("openrouter", "never-seen") is None
|
||||
|
||||
|
||||
def test_apply_history_accumulates_without_touching_circuit_state():
|
||||
model_health.apply_history("openrouter", "seeded-model", success_count=10, failure_count=2, total_reward=6.0)
|
||||
snapshot = model_health.snapshot_for("openrouter", "seeded-model")
|
||||
assert snapshot["success_count"] == 10
|
||||
assert snapshot["failure_count"] == 2
|
||||
assert snapshot["circuit_open"] is False
|
||||
|
||||
|
||||
def test_record_outcome_requires_a_model_name():
|
||||
model_health.record_outcome("openrouter", "", True, latency_ms=100)
|
||||
assert model_health.snapshot_all() == {}
|
||||
|
||||
|
||||
def test_snapshot_all_keys_are_provider_colon_model():
|
||||
model_health.record_outcome("groq", "llama-3", True)
|
||||
keys = model_health.snapshot_all().keys()
|
||||
assert "groq:llama-3" in keys
|
||||
@@ -0,0 +1,37 @@
|
||||
# retoor <retoor@molodetz.nl>
|
||||
|
||||
from devplacepy.services.openai_gateway import model_stats_query as msq
|
||||
|
||||
|
||||
def test_bucket_seconds_has_a_floor():
|
||||
assert msq._bucket_seconds(60) == 60
|
||||
assert msq._bucket_seconds(3600) == 60
|
||||
|
||||
|
||||
def test_bucket_seconds_scales_with_range():
|
||||
week = 7 * 86400
|
||||
assert msq._bucket_seconds(week) == week // 120
|
||||
|
||||
|
||||
def test_histogram_places_values_in_the_right_bucket():
|
||||
edges = (10, 20)
|
||||
labels = ("low", "mid", "high")
|
||||
result = msq._histogram([5, 15, 25, 9.999, 20], edges, labels)
|
||||
counts = {row["label"]: row["count"] for row in result}
|
||||
assert counts == {"low": 2, "mid": 1, "high": 2}
|
||||
|
||||
|
||||
def test_histogram_empty_input_is_all_zero():
|
||||
result = msq._histogram([], (200, 500), ("a", "b", "c"))
|
||||
assert all(row["count"] == 0 for row in result)
|
||||
|
||||
|
||||
def test_resolve_range_rejects_unknown_key():
|
||||
import pytest
|
||||
|
||||
with pytest.raises(ValueError):
|
||||
msq._resolve_range("not-a-real-range")
|
||||
|
||||
|
||||
def test_resolve_range_known_key():
|
||||
assert msq._resolve_range("24h") == msq.RANGE_SECONDS["24h"]
|
||||
@@ -0,0 +1,40 @@
|
||||
# retoor <retoor@molodetz.nl>
|
||||
|
||||
from devplacepy.services.openai_gateway import opencode_zen
|
||||
|
||||
|
||||
def test_opencode_id_has_the_expected_shape():
|
||||
value = opencode_zen._opencode_id()
|
||||
assert len(value) == opencode_zen._OPCODE_ID_LENGTH
|
||||
assert all(ch in opencode_zen._OPCODE_ID_ALPHABET for ch in value)
|
||||
|
||||
|
||||
def test_opencode_ids_are_not_repeated():
|
||||
ids = {opencode_zen._opencode_id() for _ in range(50)}
|
||||
assert len(ids) == 50
|
||||
|
||||
|
||||
def test_session_id_is_stable_across_calls():
|
||||
first = opencode_zen.opencode_impersonation_headers()
|
||||
second = opencode_zen.opencode_impersonation_headers()
|
||||
assert first["x-opencode-session"] == second["x-opencode-session"]
|
||||
assert first["x-opencode-session"].startswith("ses_")
|
||||
|
||||
|
||||
def test_request_id_is_fresh_every_call():
|
||||
first = opencode_zen.opencode_impersonation_headers()
|
||||
second = opencode_zen.opencode_impersonation_headers()
|
||||
assert first["x-opencode-request"] != second["x-opencode-request"]
|
||||
assert first["x-opencode-request"].startswith("msg_")
|
||||
|
||||
|
||||
def test_headers_include_the_expected_client_identity_fields():
|
||||
headers = opencode_zen.opencode_impersonation_headers()
|
||||
assert set(headers) == {
|
||||
"User-Agent",
|
||||
"x-opencode-client",
|
||||
"x-opencode-project",
|
||||
"x-opencode-session",
|
||||
"x-opencode-request",
|
||||
}
|
||||
assert headers["x-opencode-project"] == "global"
|
||||
@@ -0,0 +1,76 @@
|
||||
# retoor <retoor@molodetz.nl>
|
||||
|
||||
import asyncio
|
||||
import time
|
||||
|
||||
from tests.conftest import run_async
|
||||
from devplacepy.services.openai_gateway.reliability import (
|
||||
_retry_after_seconds,
|
||||
retry_send,
|
||||
)
|
||||
|
||||
|
||||
class FakeHeaders_reliability(dict):
|
||||
def get(self, key, default=None):
|
||||
return super().get(key.lower(), default)
|
||||
|
||||
|
||||
class FakeResp_reliability:
|
||||
def __init__(self, status_code, retry_after=None):
|
||||
self.status_code = status_code
|
||||
headers = FakeHeaders_reliability()
|
||||
if retry_after is not None:
|
||||
headers["retry-after"] = retry_after
|
||||
self.headers = headers
|
||||
|
||||
async def aclose(self):
|
||||
pass
|
||||
|
||||
|
||||
def test_retry_after_seconds_parses_numeric_header():
|
||||
assert _retry_after_seconds(FakeResp_reliability(429, "2")) == 2.0
|
||||
|
||||
|
||||
def test_retry_after_seconds_absent_returns_none():
|
||||
assert _retry_after_seconds(FakeResp_reliability(429)) is None
|
||||
|
||||
|
||||
def test_retry_after_seconds_is_capped():
|
||||
assert _retry_after_seconds(FakeResp_reliability(429, "99999")) == 30.0
|
||||
|
||||
|
||||
def test_retry_after_seconds_ignores_garbage():
|
||||
assert _retry_after_seconds(FakeResp_reliability(429, "not-a-number-or-date")) is None
|
||||
|
||||
|
||||
def test_retry_send_retries_429_and_honors_retry_after():
|
||||
responses = [FakeResp_reliability(429, "0.01"), FakeResp_reliability(200)]
|
||||
calls = []
|
||||
|
||||
async def do_call():
|
||||
calls.append(time.monotonic())
|
||||
return responses.pop(0)
|
||||
|
||||
sem = asyncio.Semaphore(1)
|
||||
resp, exc, attempts, queue_wait_ms = run_async(
|
||||
retry_send(do_call, sem, max_retries=2, backoff_ms=5000)
|
||||
)
|
||||
assert exc is None
|
||||
assert resp.status_code == 200
|
||||
assert attempts == 2
|
||||
# The 429 branch waits on the short Retry-After (0.01s), never the
|
||||
# much larger fixed 5000ms*attempt linear backoff it would otherwise use.
|
||||
assert calls[1] - calls[0] < 1.0
|
||||
|
||||
|
||||
def test_retry_send_gives_up_after_max_retries_on_429():
|
||||
async def do_call():
|
||||
return FakeResp_reliability(429, "0.001")
|
||||
|
||||
sem = asyncio.Semaphore(1)
|
||||
resp, exc, attempts, queue_wait_ms = run_async(
|
||||
retry_send(do_call, sem, max_retries=1, backoff_ms=1)
|
||||
)
|
||||
assert exc is None
|
||||
assert resp.status_code == 429
|
||||
assert attempts == 2
|
||||
@@ -5,6 +5,45 @@ from pydantic import ValidationError
|
||||
|
||||
from devplacepy.services.openai_gateway import routing as r
|
||||
from devplacepy.services.openai_gateway.usage import pricing_from_cfg
|
||||
from tests.conftest import run_async
|
||||
|
||||
|
||||
class _FakeModelsResponse:
|
||||
def __init__(self, status_code, payload=None):
|
||||
self.status_code = status_code
|
||||
self._payload = payload
|
||||
|
||||
def json(self):
|
||||
return self._payload
|
||||
|
||||
|
||||
class _FakeModelsClient:
|
||||
def __init__(self, status_code, payload=None, error=None):
|
||||
self.status_code = status_code
|
||||
self.payload = payload
|
||||
self.error = error
|
||||
self.requested_url = None
|
||||
self.headers = None
|
||||
|
||||
async def __aenter__(self):
|
||||
return self
|
||||
|
||||
async def __aexit__(self, *exc_info):
|
||||
return False
|
||||
|
||||
async def get(self, url):
|
||||
self.requested_url = url
|
||||
if self.error is not None:
|
||||
raise self.error
|
||||
return _FakeModelsResponse(self.status_code, self.payload)
|
||||
|
||||
|
||||
def _patch_stealth_client(monkeypatch, fake_client):
|
||||
def _factory(**kwargs):
|
||||
fake_client.headers = kwargs.get("headers")
|
||||
return fake_client
|
||||
|
||||
monkeypatch.setattr("devplacepy.stealth.stealth_async_client", _factory)
|
||||
|
||||
|
||||
def _cleanup(providers, models):
|
||||
@@ -262,6 +301,46 @@ def test_blank_provider_uses_default_upstream(local_db):
|
||||
_cleanup([], ["ut-default"])
|
||||
|
||||
|
||||
def test_provider_with_a_client_profile_gets_extra_headers(local_db):
|
||||
r.provider_store.set(
|
||||
r.ProviderIn(
|
||||
name="zen",
|
||||
base_url="https://opencode.ai/zen/v1/chat/completions",
|
||||
api_key="public",
|
||||
client_profile="opencode",
|
||||
)
|
||||
)
|
||||
r.model_store.set(
|
||||
r.ModelRouteIn(source_model="ut-zen", provider="zen", target_model="kimi-k3")
|
||||
)
|
||||
try:
|
||||
overlay = r.chat_overlay("ut-zen", {})
|
||||
headers = overlay["gateway_extra_request_headers"]
|
||||
assert headers["x-opencode-client"]
|
||||
assert "User-Agent" in headers
|
||||
finally:
|
||||
_cleanup(["zen"], ["ut-zen"])
|
||||
|
||||
|
||||
def test_provider_without_a_client_profile_gets_no_extra_headers(local_db):
|
||||
r.provider_store.set(
|
||||
r.ProviderIn(name="plainprov", base_url="https://up.example/v1/chat/completions")
|
||||
)
|
||||
r.model_store.set(
|
||||
r.ModelRouteIn(source_model="ut-plain", provider="plainprov", target_model="vendor/x")
|
||||
)
|
||||
try:
|
||||
overlay = r.chat_overlay("ut-plain", {})
|
||||
assert "gateway_extra_request_headers" not in overlay
|
||||
finally:
|
||||
_cleanup(["plainprov"], ["ut-plain"])
|
||||
|
||||
|
||||
def test_client_profile_rejects_unknown_value():
|
||||
with pytest.raises(ValidationError):
|
||||
r.ProviderIn(name="badprofile", client_profile="not-a-real-profile")
|
||||
|
||||
|
||||
def test_tier2_and_off_peak_fields_propagate_through_overlay(local_db):
|
||||
r.model_store.set(
|
||||
r.ModelRouteIn(
|
||||
@@ -381,3 +460,112 @@ def test_seed_publishes_molodetz_aliases(local_db):
|
||||
finally:
|
||||
r.model_store.remove("molodetz")
|
||||
r.model_store.remove("molodetz-pro")
|
||||
|
||||
|
||||
def test_models_url_from_base_swaps_chat_completions():
|
||||
assert (
|
||||
r._models_url_from_base("https://x.example/v1/chat/completions")
|
||||
== "https://x.example/v1/models"
|
||||
)
|
||||
|
||||
|
||||
def test_models_url_from_base_appends_when_no_chat_completions_suffix():
|
||||
assert r._models_url_from_base("https://x.example/v1") == "https://x.example/v1/models"
|
||||
assert r._models_url_from_base("https://x.example/v1/") == "https://x.example/v1/models"
|
||||
|
||||
|
||||
def test_models_url_from_base_blank_is_blank():
|
||||
assert r._models_url_from_base("") == ""
|
||||
assert r._models_url_from_base(" ") == ""
|
||||
|
||||
|
||||
def test_fetch_provider_models_returns_ids_on_success(local_db, monkeypatch):
|
||||
r.provider_store.set(
|
||||
r.ProviderIn(
|
||||
name="probeprov",
|
||||
base_url="https://x.example/v1/chat/completions",
|
||||
api_key="sk-probe",
|
||||
)
|
||||
)
|
||||
fake_client = _FakeModelsClient(200, {"data": [{"id": "vendor/a"}, {"id": "vendor/b"}]})
|
||||
_patch_stealth_client(monkeypatch, fake_client)
|
||||
try:
|
||||
models = run_async(r.fetch_provider_models("probeprov"))
|
||||
assert models == ["vendor/a", "vendor/b"]
|
||||
assert fake_client.requested_url == "https://x.example/v1/models"
|
||||
assert fake_client.headers == {"authorization": "Bearer sk-probe"}
|
||||
finally:
|
||||
r.provider_store.remove("probeprov")
|
||||
|
||||
|
||||
def test_fetch_provider_models_uses_default_provider_when_blank(local_db, monkeypatch):
|
||||
monkeypatch.setattr(
|
||||
r,
|
||||
"_default_provider_credentials",
|
||||
lambda: ("https://default.example/v1/chat/completions", "sk-default"),
|
||||
)
|
||||
fake_client = _FakeModelsClient(200, {"data": [{"id": "default/model"}]})
|
||||
_patch_stealth_client(monkeypatch, fake_client)
|
||||
models = run_async(r.fetch_provider_models(""))
|
||||
assert models == ["default/model"]
|
||||
assert fake_client.requested_url == "https://default.example/v1/models"
|
||||
|
||||
|
||||
def test_fetch_provider_models_none_for_unknown_provider(local_db):
|
||||
assert run_async(r.fetch_provider_models("no-such-provider")) is None
|
||||
|
||||
|
||||
def test_fetch_provider_models_none_on_non_200(local_db, monkeypatch):
|
||||
r.provider_store.set(
|
||||
r.ProviderIn(name="probeprov404", base_url="https://x.example/v1/chat/completions")
|
||||
)
|
||||
fake_client = _FakeModelsClient(404, {})
|
||||
_patch_stealth_client(monkeypatch, fake_client)
|
||||
try:
|
||||
assert run_async(r.fetch_provider_models("probeprov404")) is None
|
||||
finally:
|
||||
r.provider_store.remove("probeprov404")
|
||||
|
||||
|
||||
def test_fetch_provider_models_none_on_malformed_payload(local_db, monkeypatch):
|
||||
r.provider_store.set(
|
||||
r.ProviderIn(name="probeprovbad", base_url="https://x.example/v1/chat/completions")
|
||||
)
|
||||
fake_client = _FakeModelsClient(200, {"not_data": []})
|
||||
_patch_stealth_client(monkeypatch, fake_client)
|
||||
try:
|
||||
assert run_async(r.fetch_provider_models("probeprovbad")) is None
|
||||
finally:
|
||||
r.provider_store.remove("probeprovbad")
|
||||
|
||||
|
||||
def test_fetch_provider_models_none_on_empty_list(local_db, monkeypatch):
|
||||
r.provider_store.set(
|
||||
r.ProviderIn(name="probeprovempty", base_url="https://x.example/v1/chat/completions")
|
||||
)
|
||||
fake_client = _FakeModelsClient(200, {"data": []})
|
||||
_patch_stealth_client(monkeypatch, fake_client)
|
||||
try:
|
||||
assert run_async(r.fetch_provider_models("probeprovempty")) is None
|
||||
finally:
|
||||
r.provider_store.remove("probeprovempty")
|
||||
|
||||
|
||||
def test_fetch_provider_models_none_on_network_error(local_db, monkeypatch):
|
||||
r.provider_store.set(
|
||||
r.ProviderIn(name="probeprovdown", base_url="https://x.example/v1/chat/completions")
|
||||
)
|
||||
fake_client = _FakeModelsClient(200, error=RuntimeError("connection refused"))
|
||||
_patch_stealth_client(monkeypatch, fake_client)
|
||||
try:
|
||||
assert run_async(r.fetch_provider_models("probeprovdown")) is None
|
||||
finally:
|
||||
r.provider_store.remove("probeprovdown")
|
||||
|
||||
|
||||
def test_fetch_provider_models_none_when_provider_has_no_base_url(local_db):
|
||||
r.provider_store.set(r.ProviderIn(name="probeprovnourl", base_url=""))
|
||||
try:
|
||||
assert run_async(r.fetch_provider_models("probeprovnourl")) is None
|
||||
finally:
|
||||
r.provider_store.remove("probeprovnourl")
|
||||
|
||||
@@ -38,6 +38,15 @@ def test_upstream_capabilities_explicit_dialect_override():
|
||||
assert caps.supports_stream_options is False
|
||||
|
||||
|
||||
def test_upstream_capabilities_ollama_stream_usage_opt_in():
|
||||
caps = upstream_capabilities(
|
||||
"http://127.0.0.1:11434/api/chat", ollama_stream_usage=True
|
||||
)
|
||||
assert caps.dialect == "ollama"
|
||||
assert caps.supports_stream_options is True
|
||||
assert caps.supports_stream_usage is True
|
||||
|
||||
|
||||
def test_apply_thinking_respects_explicit_dialect_override():
|
||||
without = apply_thinking({}, "https://ai.example.com/v1/chat/completions")
|
||||
assert without["thinking"] == {"type": "disabled"}
|
||||
@@ -91,6 +100,7 @@ def test_default_disables_openrouter_and_ollama():
|
||||
assert "thinking" not in openrouter
|
||||
ollama = apply_thinking({}, "http://127.0.0.1:11434/api/chat")
|
||||
assert ollama["think"] is False
|
||||
assert ollama["reasoning_effort"] == "none"
|
||||
assert "thinking" not in ollama
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user