chore: reorganize test files into domain-specific subdirectories under tests/
Split the monolithic test directory into three tiers (unit, api, e2e) with a path-mirroring directory structure. Added corresponding Makefile targets (test-unit, test-api, test-e2e) and updated all documentation references (CLAUDE.md, README.md, testing-cicd.html, testing-framework.html, testing-make.html) to reflect the new layout and naming conventions.
This commit is contained in:
@@ -0,0 +1,177 @@
|
||||
# retoor <retoor@molodetz.nl>
|
||||
|
||||
from datetime import datetime, timezone
|
||||
from starlette.requests import Request
|
||||
from tests.conftest import BASE_URL
|
||||
from devplacepy.database import get_table, set_setting
|
||||
from devplacepy.utils import generate_uid
|
||||
from devplacepy.services.openai_gateway import GatewayService
|
||||
from devplacepy.services.openai_gateway.analytics import build_user_usage
|
||||
from devplacepy.services.devii.config import (
|
||||
build_settings,
|
||||
FIELD_AI_URL,
|
||||
FIELD_AI_MODEL,
|
||||
FIELD_AI_KEY,
|
||||
FIELD_MAX_ITERATIONS,
|
||||
FIELD_PLAN_REQUIRED,
|
||||
FIELD_VERIFY_REQUIRED,
|
||||
)
|
||||
def _now_iso_ai_usage_profile():
|
||||
return datetime.now(timezone.utc).isoformat()
|
||||
def _make_user_ai_usage_profile(role="Member"):
|
||||
username = f"aiu_{generate_uid()[:8]}"
|
||||
api_key = generate_uid()
|
||||
uid = generate_uid()
|
||||
get_table("users").insert(
|
||||
{
|
||||
"uid": uid,
|
||||
"username": username,
|
||||
"email": f"{username}@t.dev",
|
||||
"api_key": api_key,
|
||||
"role": role,
|
||||
"is_active": True,
|
||||
}
|
||||
)
|
||||
return uid, username, api_key
|
||||
def _make_request_ai_usage_profile(headers):
|
||||
raw = [(k.lower().encode(), v.encode()) for k, v in headers.items()]
|
||||
return Request(
|
||||
{
|
||||
"type": "http",
|
||||
"method": "POST",
|
||||
"path": "/openai/v1/chat/completions",
|
||||
"query_string": b"",
|
||||
"headers": raw,
|
||||
"state": {},
|
||||
}
|
||||
)
|
||||
def _seed_gateway(uid, n=3, cost=0.01, kind="user"):
|
||||
table = get_table("gateway_usage_ledger")
|
||||
for _ in range(n):
|
||||
table.insert(
|
||||
{
|
||||
"created_at": _now_iso_ai_usage_profile(),
|
||||
"owner_kind": kind,
|
||||
"owner_id": uid,
|
||||
"backend": "chat",
|
||||
"model": "molodetz",
|
||||
"success": 1,
|
||||
"status_code": 200,
|
||||
"prompt_tokens": 1000,
|
||||
"completion_tokens": 300,
|
||||
"total_tokens": 1300,
|
||||
"upstream_latency_ms": 1500.0,
|
||||
"tokens_per_second": 40.0,
|
||||
"cost_usd": cost,
|
||||
}
|
||||
)
|
||||
def _devii_cfg():
|
||||
return {
|
||||
FIELD_AI_URL: "",
|
||||
FIELD_AI_MODEL: "",
|
||||
FIELD_AI_KEY: "INTERNAL-KEY",
|
||||
FIELD_MAX_ITERATIONS: "40",
|
||||
FIELD_PLAN_REQUIRED: "1",
|
||||
FIELD_VERIFY_REQUIRED: "1",
|
||||
}
|
||||
def _goto_profile(page, username):
|
||||
page.goto(f"{BASE_URL}/profile/{username}", wait_until="domcontentloaded")
|
||||
|
||||
|
||||
def test_devii_user_session_uses_own_api_key(local_db):
|
||||
cfg = _devii_cfg()
|
||||
user_settings = build_settings(cfg, "http://x", "USER-KEY-123", "user")
|
||||
guest_settings = build_settings(cfg, "http://x", "", "guest")
|
||||
assert user_settings.ai_key == "USER-KEY-123"
|
||||
assert user_settings.platform_api_key == "USER-KEY-123"
|
||||
assert guest_settings.ai_key == "INTERNAL-KEY"
|
||||
|
||||
|
||||
def test_gateway_allows_users_by_default(local_db):
|
||||
field = next(
|
||||
f for f in GatewayService().config_fields if f.key == "gateway_allow_users"
|
||||
)
|
||||
assert field.default is True
|
||||
|
||||
|
||||
def test_gateway_attributes_calls_to_the_user(local_db):
|
||||
member_uid, _, member_key = _make_user_ai_usage_profile(role="Member")
|
||||
admin_uid, _, admin_key = _make_user_ai_usage_profile(role="Admin")
|
||||
svc = GatewayService()
|
||||
assert svc.resolve_owner(
|
||||
_make_request_ai_usage_profile({"Authorization": f"Bearer {member_key}"})
|
||||
) == ("user", member_uid)
|
||||
assert svc.resolve_owner(_make_request_ai_usage_profile({"X-API-KEY": admin_key})) == (
|
||||
"admin",
|
||||
admin_uid,
|
||||
)
|
||||
|
||||
|
||||
def test_build_user_usage_aggregates(local_db):
|
||||
uid, _, _ = _make_user_ai_usage_profile()
|
||||
_seed_gateway(uid, n=4, cost=0.02)
|
||||
out = build_user_usage(uid, hours=24)
|
||||
assert out["owner_id"] == uid
|
||||
assert out["requests"] == 4
|
||||
assert out["success"] == 4
|
||||
assert out["success_pct"] == 100.0
|
||||
assert round(out["cost"]["window_usd"], 4) == 0.08
|
||||
assert out["cost"]["projected_30d_usd"] == round(0.08 * 30, 2)
|
||||
assert out["tokens"]["total"] == 4 * 1300
|
||||
assert any(m["key"] == "molodetz" for m in out["by_model"])
|
||||
assert out["hourly"]
|
||||
|
||||
|
||||
def test_build_user_usage_isolates_users(local_db):
|
||||
mine, _, _ = _make_user_ai_usage_profile()
|
||||
other, _, _ = _make_user_ai_usage_profile()
|
||||
_seed_gateway(mine, n=2, cost=0.05)
|
||||
_seed_gateway(other, n=9, cost=0.05)
|
||||
out = build_user_usage(mine, hours=24)
|
||||
assert out["requests"] == 2
|
||||
|
||||
|
||||
def test_build_user_usage_empty_for_unknown(local_db):
|
||||
out = build_user_usage("no-such-uid-xyz", hours=24)
|
||||
assert out["requests"] == 0
|
||||
assert out["cost"]["projected_30d_usd"] == 0.0
|
||||
assert "projection" in out["notes"]
|
||||
|
||||
|
||||
def test_ai_quota_percentage_and_clamp(local_db):
|
||||
from devplacepy.services.manager import service_manager
|
||||
from devplacepy.services.devii import DeviiService
|
||||
from devplacepy.routers.profile import _ai_quota
|
||||
|
||||
if service_manager.get_service("devii") is None:
|
||||
service_manager.register(DeviiService())
|
||||
set_setting("devii_user_daily_usd", "2.0")
|
||||
try:
|
||||
half, _, _ = _make_user_ai_usage_profile()
|
||||
get_table("devii_usage_ledger").insert(
|
||||
{
|
||||
"owner_kind": "user",
|
||||
"owner_id": half,
|
||||
"created_at": _now_iso_ai_usage_profile(),
|
||||
"cost_usd": 1.0,
|
||||
}
|
||||
)
|
||||
quota = _ai_quota(half)
|
||||
assert quota is not None
|
||||
assert quota["used_pct"] == 50.0
|
||||
assert quota["turns"] == 1
|
||||
|
||||
over, _, _ = _make_user_ai_usage_profile()
|
||||
for _ in range(3):
|
||||
get_table("devii_usage_ledger").insert(
|
||||
{
|
||||
"owner_kind": "user",
|
||||
"owner_id": over,
|
||||
"created_at": _now_iso_ai_usage_profile(),
|
||||
"cost_usd": 1.0,
|
||||
}
|
||||
)
|
||||
capped = _ai_quota(over)
|
||||
assert capped["used_pct"] == 100.0
|
||||
finally:
|
||||
set_setting("devii_user_daily_usd", "1.0")
|
||||
@@ -0,0 +1,204 @@
|
||||
# retoor <retoor@molodetz.nl>
|
||||
|
||||
import json
|
||||
import requests
|
||||
from starlette.requests import Request
|
||||
from tests.conftest import BASE_URL, run_async
|
||||
from devplacepy.database import get_table, set_setting
|
||||
from devplacepy.utils import generate_uid
|
||||
import devplacepy.services.openai_gateway.gateway as gwmod
|
||||
from devplacepy.services.openai_gateway import GatewayService
|
||||
class FakeResp_openai_gateway:
|
||||
def __init__(self, status=200, payload=None, ctype="application/json", content=b""):
|
||||
self.status_code = status
|
||||
self._payload = payload
|
||||
self.text = json.dumps(payload) if payload is not None else ""
|
||||
self.headers = {"content-type": ctype}
|
||||
self.content = content
|
||||
|
||||
def json(self):
|
||||
if self._payload is None:
|
||||
raise ValueError("no json")
|
||||
return self._payload
|
||||
class FakeRequest:
|
||||
def __init__(self, method, url, json_body):
|
||||
self.method = method
|
||||
self.url = url
|
||||
self.json_body = json_body
|
||||
self.extensions = {}
|
||||
class FakeClient_openai_gateway:
|
||||
def __init__(self, *a, **k):
|
||||
self.calls = []
|
||||
|
||||
def build_request(self, method, url, headers=None, json=None, content=None):
|
||||
return FakeRequest(method, url, json)
|
||||
|
||||
async def send(self, request):
|
||||
self.calls.append((request.url, request.json_body))
|
||||
body = request.json_body or {}
|
||||
return FakeResp_openai_gateway(
|
||||
payload={
|
||||
"id": "x",
|
||||
"model": body.get("model"),
|
||||
"choices": [{"message": {"content": "hi there"}}],
|
||||
}
|
||||
)
|
||||
|
||||
async def post(self, url, headers=None, json=None, timeout=None):
|
||||
self.calls.append((url, json))
|
||||
return FakeResp_openai_gateway(
|
||||
payload={
|
||||
"id": "x",
|
||||
"model": json.get("model"),
|
||||
"choices": [{"message": {"content": "hi there"}}],
|
||||
}
|
||||
)
|
||||
|
||||
async def request(self, method, url, headers=None, content=None):
|
||||
return FakeResp_openai_gateway(payload={"ok": True})
|
||||
|
||||
async def aclose(self):
|
||||
pass
|
||||
def _make_request_openai_gateway(headers=None, cookies=None):
|
||||
headers = headers or {}
|
||||
raw = [(k.lower().encode(), v.encode()) for k, v in headers.items()]
|
||||
if cookies:
|
||||
raw.append(
|
||||
(b"cookie", "; ".join(f"{k}={v}" for k, v in cookies.items()).encode())
|
||||
)
|
||||
return Request(
|
||||
{
|
||||
"type": "http",
|
||||
"method": "POST",
|
||||
"path": "/openai/v1/chat/completions",
|
||||
"query_string": b"",
|
||||
"headers": raw,
|
||||
"state": {},
|
||||
}
|
||||
)
|
||||
def _make_admin_openai_gateway(role="Admin"):
|
||||
username = f"gw_{generate_uid()[:8]}"
|
||||
api_key = generate_uid()
|
||||
get_table("users").insert(
|
||||
{
|
||||
"uid": generate_uid(),
|
||||
"username": username,
|
||||
"email": f"{username}@t.dev",
|
||||
"api_key": api_key,
|
||||
"role": role,
|
||||
"is_active": True,
|
||||
}
|
||||
)
|
||||
return username, api_key
|
||||
def _config(page, **fields):
|
||||
page.request.post(f"{BASE_URL}/admin/services/openai/config", form=fields)
|
||||
|
||||
|
||||
def test_model_is_forced(local_db, monkeypatch):
|
||||
monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeClient_openai_gateway)
|
||||
svc = GatewayService()
|
||||
cfg = svc.effective_config()
|
||||
cfg["gateway_force_model"] = True
|
||||
cfg["gateway_model"] = "deepseek-chat"
|
||||
rt = svc.runtime()
|
||||
run_async(
|
||||
rt.handle_chat(
|
||||
{"model": "gpt-4", "messages": [{"role": "user", "content": "hi"}]},
|
||||
cfg,
|
||||
("guest", "test"),
|
||||
"test",
|
||||
)
|
||||
)
|
||||
assert rt._client.calls[-1][1]["model"] == "deepseek-chat"
|
||||
|
||||
|
||||
def test_vision_rewrites_image_to_text(local_db, monkeypatch):
|
||||
monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeClient_openai_gateway)
|
||||
svc = GatewayService()
|
||||
cfg = svc.effective_config()
|
||||
cfg["gateway_vision_enabled"] = True
|
||||
cfg["gateway_vision_key"] = "" # no key -> placeholder text, still rewrites to str
|
||||
rt = svc.runtime()
|
||||
msgs = [
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "text", "text": "what is this"},
|
||||
{"type": "image_url", "image_url": {"url": "http://x/y.png"}},
|
||||
],
|
||||
}
|
||||
]
|
||||
run_async(rt.handle_chat({"messages": msgs}, cfg, ("guest", "test"), "test"))
|
||||
sent = rt._client.calls[-1][1]["messages"][0]["content"]
|
||||
assert isinstance(sent, str) and "vision" in sent.lower()
|
||||
|
||||
|
||||
def test_streaming_emits_sse(local_db, monkeypatch):
|
||||
monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeClient_openai_gateway)
|
||||
svc = GatewayService()
|
||||
cfg = svc.effective_config()
|
||||
rt = svc.runtime()
|
||||
resp = run_async(
|
||||
rt.handle_chat(
|
||||
{"messages": [{"role": "user", "content": "hi"}], "stream": True},
|
||||
cfg,
|
||||
("guest", "test"),
|
||||
"test",
|
||||
)
|
||||
)
|
||||
|
||||
async def drain():
|
||||
out = []
|
||||
async for chunk in resp.body_iterator:
|
||||
out.append(chunk if isinstance(chunk, str) else chunk.decode())
|
||||
return "".join(out)
|
||||
|
||||
body = run_async(drain())
|
||||
assert "chat.completion.chunk" in body
|
||||
assert "[DONE]" in body
|
||||
|
||||
|
||||
def test_authorize_static_access_key(local_db):
|
||||
set_setting("gateway_require_auth", "1")
|
||||
set_setting("gateway_access_key", "topsecret")
|
||||
try:
|
||||
svc = GatewayService()
|
||||
assert svc.authorize(_make_request_openai_gateway(headers={"X-API-KEY": "topsecret"})) is True
|
||||
assert svc.authorize(_make_request_openai_gateway(headers={"X-API-KEY": "wrong"})) is False
|
||||
finally:
|
||||
set_setting("gateway_access_key", "")
|
||||
set_setting("gateway_require_auth", "1")
|
||||
|
||||
|
||||
def test_authorize_admin_and_user_toggles(local_db):
|
||||
set_setting("gateway_require_auth", "1")
|
||||
set_setting("gateway_access_key", "")
|
||||
set_setting("gateway_allow_admins", "1")
|
||||
set_setting("gateway_allow_users", "0")
|
||||
try:
|
||||
_, admin_key = _make_admin_openai_gateway(role="Admin")
|
||||
_, member_key = _make_admin_openai_gateway(role="Member")
|
||||
svc = GatewayService()
|
||||
|
||||
assert (
|
||||
svc.authorize(_make_request_openai_gateway(headers={"Authorization": f"Bearer {admin_key}"}))
|
||||
is True
|
||||
)
|
||||
assert svc.authorize(_make_request_openai_gateway(headers={"X-API-KEY": member_key})) is False
|
||||
|
||||
set_setting("gateway_allow_users", "1")
|
||||
assert svc.authorize(_make_request_openai_gateway(headers={"X-API-KEY": member_key})) is True
|
||||
finally:
|
||||
set_setting("gateway_allow_admins", "1")
|
||||
set_setting("gateway_allow_users", "1")
|
||||
set_setting("gateway_require_auth", "1")
|
||||
set_setting("gateway_access_key", "")
|
||||
|
||||
|
||||
def test_authorize_require_auth_off_is_open(local_db):
|
||||
set_setting("gateway_require_auth", "0")
|
||||
try:
|
||||
svc = GatewayService()
|
||||
assert svc.authorize(_make_request_openai_gateway()) is True
|
||||
finally:
|
||||
set_setting("gateway_require_auth", "1")
|
||||
Reference in New Issue
Block a user