forked from retoor/devplacepy
Add openai_gateway/apps.py as the single list of internal gateway apps and send X-OpenRouter-Title (plus legacy X-Title and HTTP-Referer) per app to every upstream. Label the Devii agent, bots, modifier, quiz grading, SEO metadata, community and sync callers. Shorten two app references that exceeded the 30 character limit and were recorded as default. Also includes the pending Muse Code, route health and Responses work. Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
339 lines
12 KiB
Python
339 lines
12 KiB
Python
# retoor <retoor@molodetz.nl>
|
|
|
|
from dataclasses import replace
|
|
from datetime import datetime, timezone
|
|
|
|
from devplacepy.services.openai_gateway.usage import (
|
|
USAGE_FIELDS,
|
|
Pricing,
|
|
accumulate_usage,
|
|
classify_error,
|
|
compute_cost,
|
|
new_usage_totals,
|
|
normalize_usage,
|
|
parse_usage_headers,
|
|
upstream_reported_model,
|
|
usage_metric_cards,
|
|
)
|
|
|
|
BASE_PRICING = Pricing(
|
|
chat_cache_hit_per_m=0.0028,
|
|
chat_cache_miss_per_m=0.14,
|
|
chat_output_per_m=0.28,
|
|
vision_input_per_m=0.0,
|
|
vision_output_per_m=0.0,
|
|
embed_input_per_m=0.01,
|
|
)
|
|
|
|
GATEWAY_HEADERS = {
|
|
"X-Gateway-Cost-USD": "0.00010000",
|
|
"X-Gateway-Model": "molodetz",
|
|
"X-Gateway-Prompt-Tokens": "500",
|
|
"X-Gateway-Completion-Tokens": "100",
|
|
"X-Gateway-Total-Tokens": "600",
|
|
"X-Gateway-Upstream-Latency-Ms": "400",
|
|
"X-Gateway-Total-Latency-Ms": "450",
|
|
}
|
|
|
|
|
|
class FakeResp:
|
|
def __init__(self, headers=None):
|
|
self.headers = headers if headers is not None else {}
|
|
|
|
|
|
def test_new_usage_totals_is_zeroed():
|
|
totals = new_usage_totals()
|
|
assert set(totals) == set(USAGE_FIELDS)
|
|
assert all(value == 0 for value in totals.values())
|
|
|
|
|
|
def test_accumulate_usage_sums_headers():
|
|
totals = new_usage_totals()
|
|
accumulate_usage(totals, FakeResp(GATEWAY_HEADERS))
|
|
accumulate_usage(totals, FakeResp(GATEWAY_HEADERS))
|
|
assert totals["calls"] == 2
|
|
assert totals["prompt_tokens"] == 1000
|
|
assert totals["completion_tokens"] == 200
|
|
assert totals["total_tokens"] == 1200
|
|
assert abs(totals["cost_usd"] - 0.0002) < 1e-9
|
|
assert totals["upstream_latency_ms"] == 800.0
|
|
assert totals["total_latency_ms"] == 900.0
|
|
|
|
|
|
def test_accumulate_usage_is_noop_without_headers_or_totals():
|
|
totals = new_usage_totals()
|
|
accumulate_usage(None, FakeResp(GATEWAY_HEADERS))
|
|
accumulate_usage(totals, FakeResp({}))
|
|
accumulate_usage(totals, object())
|
|
assert totals["calls"] == 0
|
|
|
|
|
|
def test_parse_usage_headers_requires_cost_header():
|
|
assert parse_usage_headers({}) is None
|
|
assert parse_usage_headers(None) is None
|
|
parsed = parse_usage_headers(GATEWAY_HEADERS)
|
|
assert parsed["calls"] == 1
|
|
assert parsed["total_tokens"] == 600
|
|
|
|
|
|
def test_usage_metric_cards_labels_and_formatting():
|
|
usage = {
|
|
"calls": 4,
|
|
"total_tokens": 9200,
|
|
"prompt_tokens": 6000,
|
|
"completion_tokens": 3200,
|
|
"cost_usd": 0.0492,
|
|
"avg_tokens": 2300.0,
|
|
"avg_cost_usd": 0.0123,
|
|
"avg_upstream_latency_ms": 4200.0,
|
|
"avg_tokens_per_second": 190.5,
|
|
}
|
|
cards = usage_metric_cards(usage)
|
|
labels = [card["label"] for card in cards]
|
|
assert labels == [
|
|
"AI calls",
|
|
"Total tokens",
|
|
"Prompt tokens",
|
|
"Completion tokens",
|
|
"Total cost",
|
|
"Avg tokens/call",
|
|
"Avg cost/call",
|
|
"Avg latency",
|
|
"Avg tokens/sec",
|
|
]
|
|
by_label = {card["label"]: card["value"] for card in cards}
|
|
assert by_label["AI calls"] == 4
|
|
assert by_label["Total cost"] == "$0.0492"
|
|
assert by_label["Avg cost/call"] == "$0.012300"
|
|
assert by_label["Avg latency"] == "4200ms"
|
|
|
|
|
|
def test_compute_cost_matches_deepseek_flat_rates_when_no_tier_configured():
|
|
norm = normalize_usage(
|
|
{
|
|
"prompt_tokens": 2000,
|
|
"completion_tokens": 100,
|
|
"prompt_cache_hit_tokens": 500,
|
|
"prompt_cache_miss_tokens": 1500,
|
|
}
|
|
)
|
|
total, input_cost, output_cost, native = compute_cost({}, norm, BASE_PRICING, "chat")
|
|
assert native is False
|
|
assert abs(input_cost - (500 / 1e6 * 0.0028 + 1500 / 1e6 * 0.14)) < 1e-12
|
|
assert abs(output_cost - (100 / 1e6 * 0.28)) < 1e-12
|
|
assert abs(total - (input_cost + output_cost)) < 1e-12
|
|
|
|
|
|
def test_compute_cost_switches_to_tier2_above_threshold():
|
|
tiered = replace(
|
|
BASE_PRICING,
|
|
chat_cache_hit_per_m_tier2=0.005,
|
|
chat_cache_miss_per_m_tier2=0.25,
|
|
chat_output_per_m_tier2=0.5,
|
|
context_tier_threshold_tokens=1000,
|
|
)
|
|
below = normalize_usage({"prompt_tokens": 900, "completion_tokens": 50})
|
|
above = normalize_usage({"prompt_tokens": 1001, "completion_tokens": 50})
|
|
_, below_input, below_output, _ = compute_cost({}, below, tiered, "chat")
|
|
_, above_input, above_output, _ = compute_cost({}, above, tiered, "chat")
|
|
assert abs(below_output - (50 / 1e6 * 0.28)) < 1e-12
|
|
assert abs(above_output - (50 / 1e6 * 0.5)) < 1e-12
|
|
assert below_input != above_input
|
|
|
|
|
|
def test_compute_cost_tier2_leaves_unset_component_at_tier1():
|
|
tiered = replace(
|
|
BASE_PRICING,
|
|
chat_output_per_m_tier2=0.5,
|
|
context_tier_threshold_tokens=100,
|
|
)
|
|
norm = normalize_usage({"prompt_tokens": 200, "completion_tokens": 10})
|
|
_, input_cost, output_cost, _ = compute_cost({}, norm, tiered, "chat")
|
|
assert abs(output_cost - (10 / 1e6 * 0.5)) < 1e-12
|
|
assert abs(input_cost - (200 / 1e6 * 0.14)) < 1e-12
|
|
|
|
|
|
def test_compute_cost_applies_off_peak_discount():
|
|
discounted = replace(
|
|
BASE_PRICING,
|
|
off_peak_start_minute=60,
|
|
off_peak_end_minute=120,
|
|
off_peak_discount_pct=50.0,
|
|
)
|
|
norm = normalize_usage({"prompt_tokens": 1000, "completion_tokens": 100})
|
|
in_window = datetime(2026, 1, 1, 1, 30, tzinfo=timezone.utc)
|
|
outside_window = datetime(2026, 1, 1, 10, 0, tzinfo=timezone.utc)
|
|
total_in, _, _, _ = compute_cost({}, norm, discounted, "chat", now=in_window)
|
|
total_out, _, _, _ = compute_cost({}, norm, discounted, "chat", now=outside_window)
|
|
total_flat, _, _, _ = compute_cost({}, norm, BASE_PRICING, "chat")
|
|
assert abs(total_out - total_flat) < 1e-12
|
|
assert abs(total_in - total_flat / 2) < 1e-12
|
|
|
|
|
|
def test_compute_cost_off_peak_wraps_past_midnight():
|
|
wrapped = replace(
|
|
BASE_PRICING,
|
|
off_peak_start_minute=23 * 60,
|
|
off_peak_end_minute=60,
|
|
off_peak_discount_pct=100.0,
|
|
)
|
|
norm = normalize_usage({"prompt_tokens": 1000, "completion_tokens": 100})
|
|
just_after_midnight = datetime(2026, 1, 1, 0, 30, tzinfo=timezone.utc)
|
|
midday = datetime(2026, 1, 1, 12, 0, tzinfo=timezone.utc)
|
|
total_wrapped, _, _, _ = compute_cost(
|
|
{}, norm, wrapped, "chat", now=just_after_midnight
|
|
)
|
|
total_midday, _, _, _ = compute_cost({}, norm, wrapped, "chat", now=midday)
|
|
assert total_wrapped == 0.0
|
|
assert total_midday > 0.0
|
|
|
|
|
|
def test_compute_cost_image_branch_flat_per_call():
|
|
image_pricing = replace(BASE_PRICING, image_per_call=0.04)
|
|
norm = normalize_usage({})
|
|
total, input_cost, output_cost, native = compute_cost(
|
|
{}, norm, image_pricing, "image"
|
|
)
|
|
assert native is False
|
|
assert output_cost == 0.0
|
|
assert abs(total - 0.04) < 1e-9
|
|
assert abs(input_cost - 0.04) < 1e-9
|
|
|
|
|
|
def test_compute_cost_image_native_cost_preferred():
|
|
from devplacepy.services.openai_gateway.usage import extract_image_usage
|
|
|
|
image_pricing = replace(BASE_PRICING, image_per_call=0.04)
|
|
norm = normalize_usage({})
|
|
usage = extract_image_usage({"usage": {"cost": 0.055}})
|
|
total, _, _, native = compute_cost(usage, norm, image_pricing, "image")
|
|
assert native is True
|
|
assert total == 0.055
|
|
|
|
|
|
def test_compute_cost_native_upstream_cost_unaffected_by_tiering():
|
|
tiered = replace(
|
|
BASE_PRICING,
|
|
chat_output_per_m_tier2=100.0,
|
|
context_tier_threshold_tokens=1,
|
|
)
|
|
norm = normalize_usage({"prompt_tokens": 1000, "completion_tokens": 100})
|
|
total, _, _, native = compute_cost({"cost": 0.05}, norm, tiered, "chat")
|
|
assert native is True
|
|
assert total == 0.05
|
|
|
|
|
|
def test_validate_app_reference_accepts_valid_slugs():
|
|
from devplacepy.services.openai_gateway.service import _validate_app_reference
|
|
|
|
assert _validate_app_reference("devplace-devii-v-1-0-0") == "devplace-devii-v-1-0-0"
|
|
assert _validate_app_reference("my-app") == "my-app"
|
|
assert _validate_app_reference("a") == "a"
|
|
assert _validate_app_reference("abcdefghijklmnopqrstuvwxyz0123") == "abcdefghijklmnopqrstuvwxyz0123"
|
|
assert _validate_app_reference("test_app.release-2") == "test_app.release-2"
|
|
|
|
|
|
def test_validate_app_reference_rejects_invalid():
|
|
from devplacepy.services.openai_gateway.service import _validate_app_reference
|
|
|
|
assert _validate_app_reference("") == "default"
|
|
assert _validate_app_reference(" ") == "default"
|
|
assert _validate_app_reference("hello world") == "default"
|
|
assert _validate_app_reference("name@domain") == "default"
|
|
assert _validate_app_reference("name/domain") == "default"
|
|
assert _validate_app_reference(None) == "default"
|
|
assert _validate_app_reference("abcdefghijklmnopqrstuvwxyz01234") == "default"
|
|
|
|
|
|
def test_validate_app_reference_strips_whitespace():
|
|
from devplacepy.services.openai_gateway.service import _validate_app_reference
|
|
|
|
assert _validate_app_reference(" my-app ") == "my-app"
|
|
assert _validate_app_reference("\tdevplace\t") == "devplace"
|
|
|
|
|
|
class DictOnlyHeaders(dict):
|
|
"""A plain mapping with no case-insensitive .get, like a hand-rolled
|
|
fake response's headers dict in a test double - exercises the
|
|
manual case-insensitive fallback scan."""
|
|
|
|
|
|
def test_upstream_reported_model_reads_exact_case_via_get():
|
|
assert upstream_reported_model({"X-Gateway-Model": "vendor/served-model"}) == (
|
|
"vendor/served-model"
|
|
)
|
|
|
|
|
|
def test_upstream_reported_model_is_case_insensitive_via_real_headers():
|
|
import httpx
|
|
|
|
headers = httpx.Headers({"x-gateway-model": "vendor/lowercase-model"})
|
|
assert upstream_reported_model(headers) == "vendor/lowercase-model"
|
|
|
|
|
|
def test_upstream_reported_model_falls_back_to_manual_scan_for_plain_dicts():
|
|
headers = DictOnlyHeaders({"x-GATEWAY-model": "vendor/mixed-case"})
|
|
assert upstream_reported_model(headers) == "vendor/mixed-case"
|
|
|
|
|
|
def test_upstream_reported_model_strips_surrounding_whitespace():
|
|
assert upstream_reported_model({"X-Gateway-Model": " vendor/spaced "}) == (
|
|
"vendor/spaced"
|
|
)
|
|
|
|
|
|
def test_upstream_reported_model_missing_header_returns_none():
|
|
assert upstream_reported_model({}) is None
|
|
assert upstream_reported_model({"Content-Type": "application/json"}) is None
|
|
|
|
|
|
def test_upstream_reported_model_handles_falsy_and_malformed_containers():
|
|
assert upstream_reported_model(None) is None
|
|
assert upstream_reported_model({}) is None
|
|
assert upstream_reported_model(object()) is None
|
|
assert upstream_reported_model(["not", "a", "mapping"]) is None
|
|
|
|
|
|
def test_upstream_reported_model_rejects_empty_or_whitespace_only():
|
|
assert upstream_reported_model({"X-Gateway-Model": ""}) is None
|
|
assert upstream_reported_model({"X-Gateway-Model": " "}) is None
|
|
|
|
|
|
def test_upstream_reported_model_rejects_non_string_value():
|
|
assert upstream_reported_model({"X-Gateway-Model": 12345}) is None
|
|
assert upstream_reported_model({"X-Gateway-Model": None}) is None
|
|
|
|
|
|
def test_upstream_reported_model_rejects_oversized_value():
|
|
huge = "x" * 201
|
|
assert upstream_reported_model({"X-Gateway-Model": huge}) is None
|
|
exactly_max = "x" * 200
|
|
assert upstream_reported_model({"X-Gateway-Model": exactly_max}) == exactly_max
|
|
|
|
|
|
def test_upstream_reported_model_rejects_control_characters():
|
|
assert upstream_reported_model({"X-Gateway-Model": "vendor\r\nInjected: header"}) is None
|
|
assert upstream_reported_model({"X-Gateway-Model": "vendor\nmodel"}) is None
|
|
assert upstream_reported_model({"X-Gateway-Model": "vendor\x00model"}) is None
|
|
|
|
|
|
def test_upstream_reported_model_get_raising_never_propagates():
|
|
class HostileHeaders:
|
|
def get(self, key, default=None):
|
|
raise RuntimeError("boom")
|
|
|
|
def items(self):
|
|
raise RuntimeError("boom too")
|
|
|
|
assert upstream_reported_model(HostileHeaders()) is None
|
|
|
|
|
|
def test_classify_error_model_unavailable_400_is_model_not_found():
|
|
body = '{"error":{"type":"server_error","message":"Upstream request failed: Model is unavailable."}}'
|
|
assert classify_error(400, None, body) == "model_not_found"
|
|
assert classify_error(404, None, "") == "model_not_found"
|
|
assert classify_error(400, None, '{"error":"bad schema"}') == "bad_request"
|
|
assert (
|
|
classify_error(400, None, "context window exceeded") == "context_length"
|
|
)
|