An upstream that rejects a key the gateway adds (e.g. thinking) now gets retried with other shapes on a parsable 400/422; the working method is stored per (provider, target model) and applied first-try from then on. Fallback attempts are re-shaped per attempt. Admin list/reset, Devii tools, audit event and ledger resolved_method included. Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
349 lines
12 KiB
Python
349 lines
12 KiB
Python
# retoor <retoor@molodetz.nl>
|
|
|
|
import http.server
|
|
import json
|
|
import socket
|
|
import socketserver
|
|
import threading
|
|
|
|
import requests
|
|
|
|
from devplacepy.database import get_table, refresh_snapshot
|
|
from tests.conftest import BASE_URL, login_user
|
|
|
|
|
|
def _promote_to_admin(username: str) -> None:
|
|
users = get_table("users")
|
|
user = users.find_one(username=username)
|
|
if user:
|
|
users.update({"uid": user["uid"], "role": "Admin"}, ["uid"])
|
|
|
|
|
|
def _admin_api_key(username: str) -> str:
|
|
refresh_snapshot()
|
|
return get_table("users").find_one(username=username)["api_key"]
|
|
|
|
|
|
def _fake_models_upstream(model_ids):
|
|
sock = socket.socket()
|
|
sock.bind(("127.0.0.1", 0))
|
|
port = sock.getsockname()[1]
|
|
sock.close()
|
|
body = json.dumps({"data": [{"id": m} for m in model_ids]}).encode()
|
|
|
|
class Handler(http.server.BaseHTTPRequestHandler):
|
|
def do_GET(self):
|
|
self.send_response(200)
|
|
self.send_header("Content-Type", "application/json")
|
|
self.end_headers()
|
|
self.wfile.write(body)
|
|
|
|
def log_message(self, *args):
|
|
pass
|
|
|
|
httpd = socketserver.TCPServer(("127.0.0.1", port), Handler)
|
|
thread = threading.Thread(target=httpd.serve_forever, daemon=True)
|
|
thread.start()
|
|
return httpd, port
|
|
|
|
|
|
def test_model_form_swaps_target_field_by_provider(page, seeded_db):
|
|
user = seeded_db["alice"]
|
|
_promote_to_admin(user["username"])
|
|
key = _admin_api_key(user["username"])
|
|
auth = {"X-API-KEY": key}
|
|
|
|
httpd, port = _fake_models_upstream(["vendor/known-a", "vendor/known-b"])
|
|
listing_provider = f"e2elisting{port}"
|
|
blind_provider = f"e2eblind{port}"
|
|
try:
|
|
requests.post(
|
|
f"{BASE_URL}/admin/gateway/providers",
|
|
json={
|
|
"name": listing_provider,
|
|
"base_url": f"http://127.0.0.1:{port}/v1/chat/completions",
|
|
},
|
|
headers=auth,
|
|
)
|
|
requests.post(
|
|
f"{BASE_URL}/admin/gateway/providers",
|
|
json={"name": blind_provider, "base_url": ""},
|
|
headers=auth,
|
|
)
|
|
|
|
login_user(page, user)
|
|
page.goto(f"{BASE_URL}/admin/gateway/models/new", wait_until="domcontentloaded")
|
|
|
|
target_input = page.locator("#gw-model-target")
|
|
target_select = page.locator("#gw-model-target-select")
|
|
target_input.wait_for(state="visible")
|
|
assert not target_select.is_visible()
|
|
|
|
page.select_option("#gw-model-provider", listing_provider)
|
|
target_select.wait_for(state="visible", timeout=10000)
|
|
assert not target_input.is_visible()
|
|
assert target_select.get_attribute("required") is not None
|
|
options = target_select.locator("option").all_inner_texts()
|
|
assert "vendor/known-a" in options
|
|
assert "vendor/known-b" in options
|
|
|
|
page.select_option("#gw-model-provider", blind_provider)
|
|
target_input.wait_for(state="visible", timeout=10000)
|
|
assert not target_select.is_visible()
|
|
finally:
|
|
httpd.shutdown()
|
|
requests.delete(
|
|
f"{BASE_URL}/admin/gateway/providers/{listing_provider}", headers=auth
|
|
)
|
|
requests.delete(
|
|
f"{BASE_URL}/admin/gateway/providers/{blind_provider}", headers=auth
|
|
)
|
|
|
|
|
|
def test_model_form_submits_selected_model_from_dropdown(page, seeded_db):
|
|
user = seeded_db["alice"]
|
|
_promote_to_admin(user["username"])
|
|
key = _admin_api_key(user["username"])
|
|
auth = {"X-API-KEY": key}
|
|
|
|
httpd, port = _fake_models_upstream(["vendor/pick-me"])
|
|
provider = f"e2esubmit{port}"
|
|
source = f"e2esource{port}"
|
|
try:
|
|
requests.post(
|
|
f"{BASE_URL}/admin/gateway/providers",
|
|
json={
|
|
"name": provider,
|
|
"base_url": f"http://127.0.0.1:{port}/v1/chat/completions",
|
|
},
|
|
headers=auth,
|
|
)
|
|
|
|
login_user(page, user)
|
|
page.goto(f"{BASE_URL}/admin/gateway/models/new", wait_until="domcontentloaded")
|
|
page.fill("#gw-model-source", source)
|
|
page.select_option("#gw-model-provider", provider)
|
|
page.locator("#gw-model-target-select").wait_for(state="visible", timeout=10000)
|
|
page.select_option("#gw-model-target-select", "vendor/pick-me")
|
|
page.click("button[type='submit']")
|
|
page.wait_for_url(f"{BASE_URL}/admin/gateway?tab=models", wait_until="domcontentloaded")
|
|
|
|
listed = requests.get(f"{BASE_URL}/admin/gateway/models", headers=auth).json()
|
|
row = next(m for m in listed["models"] if m["source_model"] == source)
|
|
assert row["target_model"] == "vendor/pick-me"
|
|
assert row["provider"] == provider
|
|
finally:
|
|
httpd.shutdown()
|
|
requests.delete(f"{BASE_URL}/admin/gateway/models/{source}", headers=auth)
|
|
requests.delete(f"{BASE_URL}/admin/gateway/providers/{provider}", headers=auth)
|
|
|
|
|
|
def _fake_probe_upstream():
|
|
sock = socket.socket()
|
|
sock.bind(("127.0.0.1", 0))
|
|
port = sock.getsockname()[1]
|
|
sock.close()
|
|
|
|
class Handler(http.server.BaseHTTPRequestHandler):
|
|
def do_POST(self):
|
|
length = int(self.headers.get("Content-Length", 0))
|
|
self.rfile.read(length)
|
|
if self.path.endswith("/images/generations"):
|
|
body = json.dumps(
|
|
{"data": [{"url": "https://img.example/h.png"}]}
|
|
).encode()
|
|
else:
|
|
body = json.dumps(
|
|
{"choices": [{"message": {"content": "OK"}}]}
|
|
).encode()
|
|
self.send_response(200)
|
|
self.send_header("Content-Type", "application/json")
|
|
self.end_headers()
|
|
self.wfile.write(body)
|
|
|
|
def log_message(self, *args):
|
|
pass
|
|
|
|
httpd = socketserver.TCPServer(("127.0.0.1", port), Handler)
|
|
thread = threading.Thread(target=httpd.serve_forever, daemon=True)
|
|
thread.start()
|
|
return httpd, port
|
|
|
|
|
|
def test_model_grid_health_badges_and_test_buttons(page, seeded_db):
|
|
user = seeded_db["alice"]
|
|
_promote_to_admin(user["username"])
|
|
key = _admin_api_key(user["username"])
|
|
auth = {"X-API-KEY": key}
|
|
|
|
httpd, port = _fake_probe_upstream()
|
|
live_provider = f"e2ehealth{port}"
|
|
dead_provider = f"e2edead{port}"
|
|
live_source = f"e2elive{port}"
|
|
dead_source = f"e2edead{port}"
|
|
try:
|
|
requests.post(
|
|
f"{BASE_URL}/admin/gateway/providers",
|
|
json={
|
|
"name": live_provider,
|
|
"base_url": f"http://127.0.0.1:{port}/v1/chat/completions",
|
|
},
|
|
headers=auth,
|
|
)
|
|
requests.post(
|
|
f"{BASE_URL}/admin/gateway/providers",
|
|
json={
|
|
"name": dead_provider,
|
|
"base_url": "http://127.0.0.1:9/v1/chat/completions",
|
|
},
|
|
headers=auth,
|
|
)
|
|
requests.post(
|
|
f"{BASE_URL}/admin/gateway/models",
|
|
json={
|
|
"source_model": live_source,
|
|
"provider": live_provider,
|
|
"target_model": "vendor/live",
|
|
"kind": "chat",
|
|
},
|
|
headers=auth,
|
|
)
|
|
requests.post(
|
|
f"{BASE_URL}/admin/gateway/models",
|
|
json={
|
|
"source_model": dead_source,
|
|
"provider": dead_provider,
|
|
"target_model": "vendor/dead",
|
|
"kind": "chat",
|
|
},
|
|
headers=auth,
|
|
)
|
|
assert (
|
|
requests.post(
|
|
f"{BASE_URL}/admin/gateway/models/{live_source}/test", headers=auth
|
|
).status_code
|
|
== 200
|
|
)
|
|
assert (
|
|
requests.post(
|
|
f"{BASE_URL}/admin/gateway/models/{dead_source}/test", headers=auth
|
|
).status_code
|
|
== 200
|
|
)
|
|
|
|
login_user(page, user)
|
|
page.goto(f"{BASE_URL}/admin/gateway?tab=models", wait_until="domcontentloaded")
|
|
grid = page.locator("table.admin-table")
|
|
grid.wait_for(state="visible")
|
|
assert page.get_by_role("button", name="Test all", exact=True).is_visible()
|
|
live_row = grid.locator("tr", has_text=live_source)
|
|
dead_row = grid.locator("tr", has_text=dead_source)
|
|
assert live_row.get_by_role("button", name="Test", exact=True).is_visible()
|
|
assert live_row.locator(".gw-badge-ok").is_visible()
|
|
assert dead_row.locator(".gw-badge-bad").is_visible()
|
|
assert "connection_error" in (dead_row.inner_text() or "")
|
|
assert "gw-row-bad" in (dead_row.get_attribute("class") or "")
|
|
finally:
|
|
httpd.shutdown()
|
|
requests.delete(f"{BASE_URL}/admin/gateway/models/{live_source}", headers=auth)
|
|
requests.delete(f"{BASE_URL}/admin/gateway/models/{dead_source}", headers=auth)
|
|
requests.delete(
|
|
f"{BASE_URL}/admin/gateway/providers/{live_provider}", headers=auth
|
|
)
|
|
requests.delete(
|
|
f"{BASE_URL}/admin/gateway/providers/{dead_provider}", headers=auth
|
|
)
|
|
|
|
|
|
def test_image_test_opens_result_modal(page, seeded_db):
|
|
user = seeded_db["alice"]
|
|
_promote_to_admin(user["username"])
|
|
key = _admin_api_key(user["username"])
|
|
auth = {"X-API-KEY": key}
|
|
|
|
httpd, port = _fake_probe_upstream()
|
|
provider = f"e2eimg{port}"
|
|
source = f"e2eimgsrc{port}"
|
|
try:
|
|
requests.post(
|
|
f"{BASE_URL}/admin/gateway/providers",
|
|
json={
|
|
"name": provider,
|
|
"base_url": f"http://127.0.0.1:{port}/v1/chat/completions",
|
|
},
|
|
headers=auth,
|
|
)
|
|
requests.post(
|
|
f"{BASE_URL}/admin/gateway/models",
|
|
json={
|
|
"source_model": source,
|
|
"provider": provider,
|
|
"target_model": "vendor/img",
|
|
"kind": "image",
|
|
},
|
|
headers=auth,
|
|
)
|
|
|
|
login_user(page, user)
|
|
page.goto(f"{BASE_URL}/admin/gateway?tab=models", wait_until="domcontentloaded")
|
|
row = page.locator("table.admin-table tr", has_text=source)
|
|
row.wait_for(state="visible")
|
|
row.get_by_role("button", name="Test", exact=True).click()
|
|
modal = page.locator("#gw-test-image-modal.visible")
|
|
modal.wait_for(state="visible", timeout=30000)
|
|
assert modal.locator(".gw-test-image").get_attribute("src") == (
|
|
"https://img.example/h.png"
|
|
)
|
|
assert source in (modal.inner_text() or "")
|
|
finally:
|
|
httpd.shutdown()
|
|
requests.delete(f"{BASE_URL}/admin/gateway/models/{source}", headers=auth)
|
|
requests.delete(f"{BASE_URL}/admin/gateway/providers/{provider}", headers=auth)
|
|
|
|
|
|
def test_model_grid_shows_a_learned_shape_badge_and_reset(page, seeded_db):
|
|
import time
|
|
|
|
from devplacepy.services.openai_gateway.learned import learned_store
|
|
from devplacepy.services.openai_gateway.shape import ShapeMethod
|
|
|
|
user = seeded_db["alice"]
|
|
_promote_to_admin(user["username"])
|
|
key = _admin_api_key(user["username"])
|
|
auth = {"X-API-KEY": key}
|
|
provider = f"e2elearn{int(time.time() * 1000)}"
|
|
source = f"{provider}-src"
|
|
try:
|
|
requests.post(
|
|
f"{BASE_URL}/admin/gateway/providers",
|
|
json={"name": provider, "base_url": "http://127.0.0.1:9/v1/chat/completions"},
|
|
headers=auth,
|
|
)
|
|
requests.post(
|
|
f"{BASE_URL}/admin/gateway/models",
|
|
json={
|
|
"source_model": source,
|
|
"provider": provider,
|
|
"target_model": "vendor/learned",
|
|
"kind": "chat",
|
|
},
|
|
headers=auth,
|
|
)
|
|
learned_store.remember(provider, "vendor/learned", ShapeMethod("none"))
|
|
time.sleep(1.5)
|
|
login_user(page, user)
|
|
page.goto(f"{BASE_URL}/admin/gateway?tab=models", wait_until="domcontentloaded")
|
|
row = page.locator("table.admin-table tr", has_text=source)
|
|
row.wait_for(state="visible")
|
|
assert "shape: none" in (row.inner_text() or "")
|
|
row.get_by_role("button", name="Reset learned", exact=True).click()
|
|
page.locator(".dialog-overlay.visible .dialog-confirm").click()
|
|
page.wait_for_url(f"{BASE_URL}/admin/gateway?tab=models", wait_until="domcontentloaded")
|
|
row = page.locator("table.admin-table tr", has_text=source)
|
|
row.wait_for(state="visible")
|
|
assert "shape: none" not in (row.inner_text() or "")
|
|
finally:
|
|
learned_store.forget(provider, "vendor/learned")
|
|
requests.delete(f"{BASE_URL}/admin/gateway/models/{source}", headers=auth)
|
|
requests.delete(f"{BASE_URL}/admin/gateway/providers/{provider}", headers=auth)
|