blindxfish/devplacepy
1
0
Fork 0
forked from retoor/devplacepy
Code Pull Requests Activity

fix: normalize unicode escape sequences and reformat multi-line expressions across codebase

Browse Source
This commit is contained in:
retoor
2026-06-09 16:48:08 +00:00
parent 66dfda88bc
commit c4f2937415
175 changed files with 12660 additions and 4175 deletions
Download Patch File Download Diff File Expand all files Collapse all files
devplacepy/services/openai_gateway/analytics.py
+222 -81
View File
@@ -30,7 +30,15 @@ def _iso(moment: datetime) -> str:
def _pset(values: list[float]) -> dict:
if not values:
return {"avg": 0.0, "p50": 0.0, "p90": 0.0, "p95": 0.0, "p99": 0.0, "max": 0.0, "count": 0}
return {
"avg": 0.0,
"p50": 0.0,
"p90": 0.0,
"p95": 0.0,
"p99": 0.0,
"max": 0.0,
"count": 0,
}
ordered = sorted(values)
return {
"avg": round(sum(ordered) / len(ordered), 3),
@@ -51,8 +59,16 @@ def _top_group(rows: list[dict], key_fn, top_n: int) -> list[dict]:
agg: dict = {}
for r in rows:
key = key_fn(r)
bucket = agg.setdefault(key, {"key": key, "requests": 0, "success": 0,
"cost_usd": 0.0, "total_tokens": 0})
bucket = agg.setdefault(
key,
{
"key": key,
"requests": 0,
"success": 0,
"cost_usd": 0.0,
"total_tokens": 0,
},
)
bucket["requests"] += 1
bucket["success"] += int(r.get("success") or 0)
bucket["cost_usd"] += float(r.get("cost_usd") or 0)
@@ -64,19 +80,24 @@ def _top_group(rows: list[dict], key_fn, top_n: int) -> list[dict]:
def _ledger_rows(cutoff: str) -> list[dict]:
return list(db.query(
f"SELECT * FROM {GATEWAY_LEDGER} WHERE created_at >= :cutoff ORDER BY created_at",
cutoff=cutoff,
))
return list(
db.query(
f"SELECT * FROM {GATEWAY_LEDGER} WHERE created_at >= :cutoff ORDER BY created_at",
cutoff=cutoff,
)
)
def _concurrency(cutoff: str) -> dict:
if GATEWAY_CONCURRENCY not in db.tables:
return {"peak": 0, "avg": 0.0, "p95": 0.0, "samples": 0}
values = [int(r["in_flight"] or 0) for r in db.query(
f"SELECT in_flight FROM {GATEWAY_CONCURRENCY} WHERE created_at >= :cutoff",
cutoff=cutoff,
)]
values = [
int(r["in_flight"] or 0)
for r in db.query(
f"SELECT in_flight FROM {GATEWAY_CONCURRENCY} WHERE created_at >= :cutoff",
cutoff=cutoff,
)
]
if not values:
return {"peak": 0, "avg": 0.0, "p95": 0.0, "samples": 0}
ordered = sorted(values)
@@ -93,13 +114,20 @@ def empty_payload(hours: int = 48) -> dict:
"window_hours": hours,
"generated_at": _iso(_now()),
"requests": 0,
"volume": {}, "tokens": {}, "latency": {}, "errors": {}, "cost": {}, "behavior": {},
"volume": {},
"tokens": {},
"latency": {},
"errors": {},
"cost": {},
"behavior": {},
"hourly": [],
"notes": {"ttft": "not available: gateway forwards non-streaming upstream"},
}
def build_analytics(hours: int = 48, top_n: int = 10, pricing: Optional[Pricing] = None) -> dict:
def build_analytics(
hours: int = 48, top_n: int = 10, pricing: Optional[Pricing] = None
) -> dict:
if GATEWAY_LEDGER not in db.tables:
return empty_payload(hours)
hours = max(1, min(hours, MAX_WINDOW_HOURS))
@@ -118,7 +146,9 @@ def build_analytics(hours: int = 48, top_n: int = 10, pricing: Optional[Pricing]
minute_counts: dict = {}
for r in rows:
minute_counts[r["created_at"][:16]] = minute_counts.get(r["created_at"][:16], 0) + 1
minute_counts[r["created_at"][:16]] = (
minute_counts.get(r["created_at"][:16], 0) + 1
)
peak_req_per_min = max(minute_counts.values()) if minute_counts else 0
first_hour: dict = {}
@@ -131,8 +161,14 @@ def build_analytics(hours: int = 48, top_n: int = 10, pricing: Optional[Pricing]
total_cost = sum(float(r.get("cost_usd") or 0) for r in rows)
input_cost = sum(float(r.get("input_cost_usd") or 0) for r in rows)
output_cost = sum(float(r.get("output_cost_usd") or 0) for r in rows)
cost_this_hour = sum(float(r.get("cost_usd") or 0) for r in rows if r["created_at"][:13] == hour_start)
cost_24h = sum(float(r.get("cost_usd") or 0) for r in rows if r["created_at"] >= day_cutoff)
cost_this_hour = sum(
float(r.get("cost_usd") or 0)
for r in rows
if r["created_at"][:13] == hour_start
)
cost_24h = sum(
float(r.get("cost_usd") or 0) for r in rows if r["created_at"] >= day_cutoff
)
prompt_total = sum(int(r.get("prompt_tokens") or 0) for r in rows)
completion_total = sum(int(r.get("completion_tokens") or 0) for r in rows)
@@ -144,10 +180,18 @@ def build_analytics(hours: int = 48, top_n: int = 10, pricing: Optional[Pricing]
caching_savings = 0.0
if pricing is not None:
rate_delta = pricing.chat_cache_miss_per_m - pricing.chat_cache_hit_per_m
chat_cache_hits = sum(int(r.get("cache_hit_tokens") or 0) for r in rows if r.get("backend") == "chat")
chat_cache_hits = sum(
int(r.get("cache_hit_tokens") or 0)
for r in rows
if r.get("backend") == "chat"
)
caching_savings = chat_cache_hits / PER_MILLION * rate_delta
util_values = [float(r["context_utilization"]) for r in rows if r.get("context_utilization") is not None]
util_values = [
float(r["context_utilization"])
for r in rows
if r.get("context_utilization") is not None
]
owner_counts: dict = {}
for r in rows:
@@ -161,7 +205,8 @@ def build_analytics(hours: int = 48, top_n: int = 10, pricing: Optional[Pricing]
ua_counts[ua] = ua_counts.get(ua, 0) + 1
user_agents = sorted(
[{"key": k, "requests": v} for k, v in ua_counts.items()],
key=lambda x: x["requests"], reverse=True,
key=lambda x: x["requests"],
reverse=True,
)[:top_n]
error_categories: dict = {}
@@ -172,7 +217,9 @@ def build_analytics(hours: int = 48, top_n: int = 10, pricing: Optional[Pricing]
count_4xx = sum(1 for r in rows if 400 <= int(r.get("status_code") or 0) < 500)
count_5xx = sum(1 for r in rows if int(r.get("status_code") or 0) >= 500)
temperatures = [float(r["temperature"]) for r in rows if r.get("temperature") is not None]
temperatures = [
float(r["temperature"]) for r in rows if r.get("temperature") is not None
]
top_ps = [float(r["top_p"]) for r in rows if r.get("top_p") is not None]
hourly = _hourly(rows, first_hour)
@@ -184,9 +231,13 @@ def build_analytics(hours: int = 48, top_n: int = 10, pricing: Optional[Pricing]
"requests_per_hour": round(requests / hours, 2),
"peak_requests_per_minute": peak_req_per_min,
"by_model": _top_group(rows, lambda r: r.get("model") or "unknown", top_n),
"by_endpoint": _top_group(rows, lambda r: r.get("endpoint") or "unknown", top_n),
"by_endpoint": _top_group(
rows, lambda r: r.get("endpoint") or "unknown", top_n
),
"by_backend": _top_group(rows, lambda r: r.get("backend") or "unknown", 0),
"by_caller": _top_group(rows, lambda r: f"{r.get('owner_kind')}:{r.get('owner_id')}", top_n),
"by_caller": _top_group(
rows, lambda r: f"{r.get('owner_kind')}:{r.get('owner_id')}", top_n
),
"concurrency": _concurrency(cutoff),
}
@@ -197,17 +248,27 @@ def build_analytics(hours: int = 48, top_n: int = 10, pricing: Optional[Pricing]
"cache_hit_total": cache_hit_total,
"cache_miss_total": cache_miss_total,
"reasoning_total": reasoning_total,
"cache_hit_rate": round(cache_hit_total / prompt_total, 4) if prompt_total else 0.0,
"input_output_ratio": round(prompt_total / completion_total, 3) if completion_total else 0.0,
"cache_hit_rate": round(cache_hit_total / prompt_total, 4)
if prompt_total
else 0.0,
"input_output_ratio": round(prompt_total / completion_total, 3)
if completion_total
else 0.0,
"tokens_per_hour": round(total_tokens / hours, 1),
"context_utilization_avg": round(sum(util_values) / len(util_values), 4) if util_values else None,
"context_utilization_avg": round(sum(util_values) / len(util_values), 4)
if util_values
else None,
"prompt": _pset(_positive(rows, "prompt_tokens")),
"completion": _pset(_positive(rows, "completion_tokens")),
"total_per_request": _pset(_positive(rows, "total_tokens")),
}
avg_upstream = sum(_positive(rows, "upstream_latency_ms")) / max(len(_positive(rows, "upstream_latency_ms")), 1)
avg_total = sum(_positive(rows, "total_latency_ms")) / max(len(_positive(rows, "total_latency_ms")), 1)
avg_upstream = sum(_positive(rows, "upstream_latency_ms")) / max(
len(_positive(rows, "upstream_latency_ms")), 1
)
avg_total = sum(_positive(rows, "total_latency_ms")) / max(
len(_positive(rows, "total_latency_ms")), 1
)
latency = {
"upstream_ms": _pset(_positive(rows, "upstream_latency_ms")),
"gateway_overhead_ms": _pset(_positive(rows, "gateway_overhead_ms")),
@@ -215,8 +276,12 @@ def build_analytics(hours: int = 48, top_n: int = 10, pricing: Optional[Pricing]
"connect_ms": _pset(_positive(rows, "connect_ms")),
"total_ms": _pset(_positive(rows, "total_latency_ms")),
"tokens_per_second": _pset(_positive(rows, "tokens_per_second")),
"gateway_overhead_share": round((avg_total - avg_upstream) / avg_total, 4) if avg_total else 0.0,
"upstream_availability_pct": round(success / requests * 100, 2) if requests else 0.0,
"gateway_overhead_share": round((avg_total - avg_upstream) / avg_total, 4)
if avg_total
else 0.0,
"upstream_availability_pct": round(success / requests * 100, 2)
if requests
else 0.0,
"ttft_ms": None,
"inter_token_ms": None,
}
@@ -240,10 +305,20 @@ def build_analytics(hours: int = 48, top_n: int = 10, pricing: Optional[Pricing]
"this_hour_usd": round(cost_this_hour, 6),
"last_24h_usd": round(cost_24h, 6),
"projected_monthly_usd": round(cost_24h * 30, 2),
"effective_per_1k_tokens_usd": round(total_cost / total_tokens * 1000, 6) if total_tokens else 0.0,
"effective_per_1k_tokens_usd": round(total_cost / total_tokens * 1000, 6)
if total_tokens
else 0.0,
"caching_savings_usd": round(caching_savings, 6),
"by_model": [{"key": b["key"], "cost_usd": b["cost_usd"]} for b in _top_group(rows, lambda r: r.get("model") or "unknown", top_n)],
"by_caller": [{"key": b["key"], "cost_usd": b["cost_usd"]} for b in _top_group(rows, lambda r: f"{r.get('owner_kind')}:{r.get('owner_id')}", top_n)],
"by_model": [
{"key": b["key"], "cost_usd": b["cost_usd"]}
for b in _top_group(rows, lambda r: r.get("model") or "unknown", top_n)
],
"by_caller": [
{"key": b["key"], "cost_usd": b["cost_usd"]}
for b in _top_group(
rows, lambda r: f"{r.get('owner_kind')}:{r.get('owner_id')}", top_n
)
],
}
behavior = {
@@ -253,11 +328,14 @@ def build_analytics(hours: int = 48, top_n: int = 10, pricing: Optional[Pricing]
"p95": round(percentile(per_owner, 0.95), 2) if per_owner else 0.0,
"max": per_owner[-1] if per_owner else 0,
},
"avg_temperature": round(sum(temperatures) / len(temperatures), 3) if temperatures else None,
"avg_temperature": round(sum(temperatures) / len(temperatures), 3)
if temperatures
else None,
"avg_top_p": round(sum(top_ps) / len(top_ps), 3) if top_ps else None,
"tool_call_requests": sum(int(r.get("has_tools") or 0) for r in rows),
"streaming_requests": sum(int(r.get("stream_requested") or 0) for r in rows),
"non_streaming_requests": requests - sum(int(r.get("stream_requested") or 0) for r in rows),
"non_streaming_requests": requests
- sum(int(r.get("stream_requested") or 0) for r in rows),
"user_agents": user_agents,
}
@@ -281,12 +359,23 @@ def _hourly(rows: list[dict], first_hour: dict) -> list[dict]:
for r in rows:
hour = r["created_at"][:13]
owner = f"{r.get('owner_kind')}:{r.get('owner_id')}"
bucket = buckets.setdefault(hour, {
"hour": hour, "requests": 0, "success": 0, "failed": 0,
"prompt_tokens": 0, "completion_tokens": 0, "total_tokens": 0,
"cost_usd": 0.0, "latency_sum": 0.0, "latency_n": 0,
"owners": set(), "new_owners": 0,
})
bucket = buckets.setdefault(
hour,
{
"hour": hour,
"requests": 0,
"success": 0,
"failed": 0,
"prompt_tokens": 0,
"completion_tokens": 0,
"total_tokens": 0,
"cost_usd": 0.0,
"latency_sum": 0.0,
"latency_n": 0,
"owners": set(),
"new_owners": 0,
},
)
bucket["requests"] += 1
if int(r.get("success") or 0):
bucket["success"] += 1
@@ -303,10 +392,16 @@ def _hourly(rows: list[dict], first_hour: dict) -> list[dict]:
bucket["owners"].add(owner)
for hour, bucket in buckets.items():
bucket["new_owners"] = sum(1 for owner in bucket["owners"] if first_hour.get(owner) == hour)
bucket["new_owners"] = sum(
1 for owner in bucket["owners"] if first_hour.get(owner) == hour
)
bucket["active_owners"] = len(bucket["owners"])
bucket["returning_owners"] = bucket["active_owners"] - bucket["new_owners"]
bucket["avg_latency_ms"] = round(bucket["latency_sum"] / bucket["latency_n"], 1) if bucket["latency_n"] else 0.0
bucket["avg_latency_ms"] = (
round(bucket["latency_sum"] / bucket["latency_n"], 1)
if bucket["latency_n"]
else 0.0
)
bucket["cost_usd"] = round(bucket["cost_usd"], 6)
del bucket["owners"], bucket["latency_sum"], bucket["latency_n"]
@@ -324,14 +419,21 @@ def empty_user_usage(owner_id: str, hours: int = 24) -> dict:
"success_pct": 0.0,
"error_pct": 0.0,
"tokens": {"prompt": 0, "completion": 0, "total": 0},
"cost": {"window_usd": 0.0, "per_hour_usd": 0.0, "per_request_usd": 0.0, "projected_30d_usd": 0.0},
"cost": {
"window_usd": 0.0,
"per_hour_usd": 0.0,
"per_request_usd": 0.0,
"projected_30d_usd": 0.0,
},
"latency": {"avg_ms": 0.0, "avg_tps": 0.0},
"first_used": None,
"last_used": None,
"by_model": [],
"by_backend": [],
"hourly": [],
"notes": {"projection": "30-day projection extrapolates the full 24h spend (24h cost x 30)"},
"notes": {
"projection": "30-day projection extrapolates the full 24h spend (24h cost x 30)"
},
}
@@ -339,7 +441,9 @@ def _user_hourly(rows: list[dict]) -> list[dict]:
buckets: dict = {}
for r in rows:
hour = r["created_at"][:13]
bucket = buckets.setdefault(hour, {"hour": hour, "requests": 0, "cost_usd": 0.0, "total_tokens": 0})
bucket = buckets.setdefault(
hour, {"hour": hour, "requests": 0, "cost_usd": 0.0, "total_tokens": 0}
)
bucket["requests"] += 1
bucket["cost_usd"] += float(r.get("cost_usd") or 0)
bucket["total_tokens"] += int(r.get("total_tokens") or 0)
@@ -349,16 +453,21 @@ def _user_hourly(rows: list[dict]) -> list[dict]:
return out
def build_user_usage(owner_id: str, hours: int = 24, pricing: Optional[Pricing] = None) -> dict:
def build_user_usage(
owner_id: str, hours: int = 24, pricing: Optional[Pricing] = None
) -> dict:
hours = max(1, min(hours, MAX_WINDOW_HOURS))
if not owner_id or GATEWAY_LEDGER not in db.tables:
return empty_user_usage(owner_id, hours)
now = _now()
cutoff = _iso(now - timedelta(hours=hours))
rows = list(db.query(
f"SELECT * FROM {GATEWAY_LEDGER} WHERE owner_id = :oid AND created_at >= :cutoff ORDER BY created_at",
oid=owner_id, cutoff=cutoff,
))
rows = list(
db.query(
f"SELECT * FROM {GATEWAY_LEDGER} WHERE owner_id = :oid AND created_at >= :cutoff ORDER BY created_at",
oid=owner_id,
cutoff=cutoff,
)
)
if not rows:
return empty_user_usage(owner_id, hours)
@@ -385,7 +494,11 @@ def build_user_usage(owner_id: str, hours: int = 24, pricing: Optional[Pricing]
"failed": failed,
"success_pct": round(success / requests * 100, 1) if requests else 0.0,
"error_pct": round(failed / requests * 100, 1) if requests else 0.0,
"tokens": {"prompt": prompt_total, "completion": completion_total, "total": total_tokens},
"tokens": {
"prompt": prompt_total,
"completion": completion_total,
"total": total_tokens,
},
"cost": {
"window_usd": round(total_cost, 6),
"per_hour_usd": round(cost_per_hour, 6),
@@ -398,50 +511,74 @@ def build_user_usage(owner_id: str, hours: int = 24, pricing: Optional[Pricing]
"by_model": _top_group(rows, lambda r: r.get("model") or "unknown", 0),
"by_backend": _top_group(rows, lambda r: r.get("backend") or "unknown", 0),
"hourly": _user_hourly(rows),
"notes": {"projection": "30-day projection extrapolates the full 24h spend (24h cost x 30)"},
"notes": {
"projection": "30-day projection extrapolates the full 24h spend (24h cost x 30)"
},
}
def summary_metrics() -> dict:
zero = {"requests": 0, "success_pct": 0.0, "error_pct": 0.0, "cost_hour": 0.0,
"cost_24h": 0.0, "tokens_24h": 0, "avg_latency_ms": 0.0, "avg_tps": 0.0,
"peak_concurrency": 0, "top_model": "-", "top_caller": "-"}
zero = {
"requests": 0,
"success_pct": 0.0,
"error_pct": 0.0,
"cost_hour": 0.0,
"cost_24h": 0.0,
"tokens_24h": 0,
"avg_latency_ms": 0.0,
"avg_tps": 0.0,
"peak_concurrency": 0,
"top_model": "-",
"top_caller": "-",
}
if GATEWAY_LEDGER not in db.tables:
return zero
now = _now()
cutoff = _iso(now - timedelta(hours=24))
hour_start = now.strftime("%Y-%m-%dT%H")
agg = list(db.query(
f"SELECT COUNT(*) AS requests, COALESCE(SUM(success),0) AS ok, "
f"COALESCE(SUM(cost_usd),0) AS cost, COALESCE(SUM(total_tokens),0) AS tokens, "
f"COALESCE(AVG(upstream_latency_ms),0) AS avg_lat, "
f"COALESCE(AVG(NULLIF(tokens_per_second,0)),0) AS avg_tps "
f"FROM {GATEWAY_LEDGER} WHERE created_at >= :cutoff",
cutoff=cutoff,
))
agg = list(
db.query(
f"SELECT COUNT(*) AS requests, COALESCE(SUM(success),0) AS ok, "
f"COALESCE(SUM(cost_usd),0) AS cost, COALESCE(SUM(total_tokens),0) AS tokens, "
f"COALESCE(AVG(upstream_latency_ms),0) AS avg_lat, "
f"COALESCE(AVG(NULLIF(tokens_per_second,0)),0) AS avg_tps "
f"FROM {GATEWAY_LEDGER} WHERE created_at >= :cutoff",
cutoff=cutoff,
)
)
row = agg[0] if agg else {}
requests = int(row.get("requests") or 0)
if not requests:
return zero
ok = int(row.get("ok") or 0)
hour_rows = list(db.query(
f"SELECT COALESCE(SUM(cost_usd),0) AS c FROM {GATEWAY_LEDGER} WHERE created_at >= :h",
h=hour_start,
))
top_model = list(db.query(
f"SELECT model, COUNT(*) AS n FROM {GATEWAY_LEDGER} WHERE created_at >= :cutoff "
f"GROUP BY model ORDER BY n DESC LIMIT 1", cutoff=cutoff,
))
top_caller = list(db.query(
f"SELECT owner_kind, owner_id, COUNT(*) AS n FROM {GATEWAY_LEDGER} WHERE created_at >= :cutoff "
f"GROUP BY owner_kind, owner_id ORDER BY n DESC LIMIT 1", cutoff=cutoff,
))
hour_rows = list(
db.query(
f"SELECT COALESCE(SUM(cost_usd),0) AS c FROM {GATEWAY_LEDGER} WHERE created_at >= :h",
h=hour_start,
)
)
top_model = list(
db.query(
f"SELECT model, COUNT(*) AS n FROM {GATEWAY_LEDGER} WHERE created_at >= :cutoff "
f"GROUP BY model ORDER BY n DESC LIMIT 1",
cutoff=cutoff,
)
)
top_caller = list(
db.query(
f"SELECT owner_kind, owner_id, COUNT(*) AS n FROM {GATEWAY_LEDGER} WHERE created_at >= :cutoff "
f"GROUP BY owner_kind, owner_id ORDER BY n DESC LIMIT 1",
cutoff=cutoff,
)
)
peak = 0
if GATEWAY_CONCURRENCY in db.tables:
peak_rows = list(db.query(
f"SELECT COALESCE(MAX(in_flight),0) AS m FROM {GATEWAY_CONCURRENCY} WHERE created_at >= :cutoff",
cutoff=cutoff,
))
peak_rows = list(
db.query(
f"SELECT COALESCE(MAX(in_flight),0) AS m FROM {GATEWAY_CONCURRENCY} WHERE created_at >= :cutoff",
cutoff=cutoff,
)
)
peak = int(peak_rows[0]["m"]) if peak_rows else 0
return {
"requests": requests,
@@ -453,6 +590,10 @@ def summary_metrics() -> dict:
"avg_latency_ms": round(float(row.get("avg_lat") or 0), 1),
"avg_tps": round(float(row.get("avg_tps") or 0), 2),
"peak_concurrency": peak,
"top_model": top_model[0]["model"] if top_model and top_model[0].get("model") else "-",
"top_caller": f"{top_caller[0]['owner_kind']}:{top_caller[0]['owner_id']}" if top_caller else "-",
"top_model": top_model[0]["model"]
if top_model and top_model[0].get("model")
else "-",
"top_caller": f"{top_caller[0]['owner_kind']}:{top_caller[0]['owner_id']}"
if top_caller
else "-",
}
devplacepy/services/openai_gateway/gateway.py
+207 -55
View File
@@ -36,24 +36,30 @@ def _fake_stream(data: dict, model: str):
reasoning_content = msg.get("reasoning_content") or ""
def _chunk(delta: dict, finish: Optional[str] = None) -> str:
return "data: " + json.dumps({
"id": chunk_id,
"object": "chat.completion.chunk",
"created": created,
"model": out_model,
"choices": [{"index": 0, "delta": delta, "finish_reason": finish}],
}) + "\n\n"
return (
"data: "
+ json.dumps(
{
"id": chunk_id,
"object": "chat.completion.chunk",
"created": created,
"model": out_model,
"choices": [{"index": 0, "delta": delta, "finish_reason": finish}],
}
)
+ "\n\n"
)
async def gen():
yield _chunk({"role": "assistant"})
if reasoning_content:
for i in range(0, len(reasoning_content), 50):
yield _chunk({"reasoning_content": reasoning_content[i:i + 50]})
yield _chunk({"reasoning_content": reasoning_content[i : i + 50]})
if tool_calls:
yield _chunk({"tool_calls": tool_calls})
elif content:
for i in range(0, len(content), 50):
yield _chunk({"content": content[i:i + 50]})
yield _chunk({"content": content[i : i + 50]})
yield _chunk({}, finish="tool_calls" if tool_calls else "stop")
yield "data: [DONE]\n\n"
@@ -66,7 +72,9 @@ def _connect_tracer(holder: dict):
async def trace(name: str, info: dict) -> None:
if name.endswith("connect_tcp.started") or name.endswith("start_tls.started"):
started[name] = time.monotonic()
elif name.endswith("connect_tcp.complete") or name.endswith("start_tls.complete"):
elif name.endswith("connect_tcp.complete") or name.endswith(
"start_tls.complete"
):
begin = started.get(name.replace(".complete", ".started"))
if begin is not None:
holder["ms"] += (time.monotonic() - begin) * 1000
@@ -83,7 +91,9 @@ class GatewayRuntime:
self._vision_cache: Optional[VisionCache] = None
self._vision_cache_size = -1
self._ledger = GatewayUsageLedger()
self._breaker = CircuitBreaker(config.CIRCUIT_THRESHOLD_DEFAULT, config.CIRCUIT_COOLDOWN_SECONDS_DEFAULT)
self._breaker = CircuitBreaker(
config.CIRCUIT_THRESHOLD_DEFAULT, config.CIRCUIT_COOLDOWN_SECONDS_DEFAULT
)
self.requests = 0
self.errors = 0
self.in_flight = 0
@@ -95,9 +105,15 @@ class GatewayRuntime:
def _ensure(self, cfg: dict):
instances = max(1, cfg["gateway_instances"])
timeout = max(1, cfg["gateway_timeout"])
if self._client is None or instances != self._instances or timeout != self._timeout:
if (
self._client is None
or instances != self._instances
or timeout != self._timeout
):
old = self._client
limits = httpx.Limits(max_connections=instances, max_keepalive_connections=instances)
limits = httpx.Limits(
max_connections=instances, max_keepalive_connections=instances
)
self._client = httpx.AsyncClient(timeout=float(timeout), limits=limits)
self._sem = asyncio.Semaphore(instances)
self._instances = instances
@@ -108,7 +124,9 @@ class GatewayRuntime:
if self._vision_cache is None or size != self._vision_cache_size:
self._vision_cache = VisionCache(size)
self._vision_cache_size = size
self._breaker.configure(cfg["gateway_circuit_threshold"], cfg["gateway_circuit_cooldown_seconds"])
self._breaker.configure(
cfg["gateway_circuit_threshold"], cfg["gateway_circuit_cooldown_seconds"]
)
return self._client, self._sem
async def aclose(self) -> None:
@@ -117,9 +135,17 @@ class GatewayRuntime:
self._client = None
self._instances = 0
async def _send(self, client, sem, method, url, headers, cfg, log, json_body=None, content=None):
timing = {"queue_wait_ms": 0.0, "upstream_latency_ms": 0.0, "connect_ms": 0.0,
"retries_attempted": 0, "retry_succeeded": False, "circuit_open": False}
async def _send(
self, client, sem, method, url, headers, cfg, log, json_body=None, content=None
):
timing = {
"queue_wait_ms": 0.0,
"upstream_latency_ms": 0.0,
"connect_ms": 0.0,
"retries_attempted": 0,
"retry_succeeded": False,
"circuit_open": False,
}
if not self._breaker.allow():
timing["circuit_open"] = True
log("circuit breaker open, rejecting upstream call")
@@ -135,18 +161,27 @@ class GatewayRuntime:
exc = None
try:
async with sem:
timing["queue_wait_ms"] = round((time.monotonic() - wait_start) * 1000, 3)
timing["queue_wait_ms"] = round(
(time.monotonic() - wait_start) * 1000, 3
)
async def do_call():
request = client.build_request(method, url, headers=headers,
json=json_body, content=content)
request = client.build_request(
method, url, headers=headers, json=json_body, content=content
)
request.extensions["trace"] = _connect_tracer(connect_holder)
return await client.send(request)
send_start = time.monotonic()
resp, exc, attempts = await retry_send(
do_call, cfg["gateway_max_retries"], cfg["gateway_retry_backoff_ms"], log)
timing["upstream_latency_ms"] = round((time.monotonic() - send_start) * 1000, 3)
do_call,
cfg["gateway_max_retries"],
cfg["gateway_retry_backoff_ms"],
log,
)
timing["upstream_latency_ms"] = round(
(time.monotonic() - send_start) * 1000, 3
)
finally:
self.in_flight -= 1
timing["connect_ms"] = round(connect_holder["ms"], 3)
@@ -166,7 +201,9 @@ class GatewayRuntime:
timing["retry_succeeded"] = attempts > 1
return resp, None, timing
async def handle_chat(self, body: dict, cfg: dict, owner: tuple, user_agent: str, log=None):
async def handle_chat(
self, body: dict, cfg: dict, owner: tuple, user_agent: str, log=None
):
log = log or (lambda message: None)
client, sem = self._ensure(cfg)
pricing = pricing_from_cfg(cfg)
@@ -177,8 +214,13 @@ class GatewayRuntime:
if cfg["gateway_vision_enabled"]:
augmenter = VisionAugmenter(
cfg["gateway_vision_url"], cfg["gateway_vision_model"], cfg["gateway_vision_key"],
self._vision_cache, ledger=self._ledger, owner=owner, pricing=pricing,
cfg["gateway_vision_url"],
cfg["gateway_vision_model"],
cfg["gateway_vision_key"],
self._vision_cache,
ledger=self._ledger,
owner=owner,
pricing=pricing,
context_map=context_map,
)
messages = await augmenter.augment_messages(client, messages)
@@ -200,21 +242,45 @@ class GatewayRuntime:
if cfg["gateway_api_key"]:
headers["Authorization"] = f"Bearer {cfg['gateway_api_key']}"
else:
log("No upstream API key configured (gateway_api_key / DEEPSEEK_API_KEY / OPENROUTER_API_KEY); upstream will likely reject the request")
log(
"No upstream API key configured (gateway_api_key / DEEPSEEK_API_KEY / OPENROUTER_API_KEY); upstream will likely reject the request"
)
resp, exc, timing = await self._send(
client, sem, "POST", cfg["gateway_upstream_url"], headers, cfg, log, json_body=payload)
client,
sem,
"POST",
cfg["gateway_upstream_url"],
headers,
cfg,
log,
json_body=payload,
)
base = {
"owner_kind": owner[0], "owner_id": owner[1], "backend": "chat",
"endpoint": "chat/completions", "model": model, "user_agent": user_agent,
**params, **timing,
"owner_kind": owner[0],
"owner_id": owner[1],
"backend": "chat",
"endpoint": "chat/completions",
"model": model,
"user_agent": user_agent,
**params,
**timing,
}
def finalize(status_code, success, category, usage=None):
base["total_latency_ms"] = round((time.monotonic() - handle_start) * 1000, 3)
base["gateway_overhead_ms"] = round(max(
base["total_latency_ms"] - timing["upstream_latency_ms"] - timing["queue_wait_ms"], 0.0), 3)
base["total_latency_ms"] = round(
(time.monotonic() - handle_start) * 1000, 3
)
base["gateway_overhead_ms"] = round(
max(
base["total_latency_ms"]
- timing["upstream_latency_ms"]
- timing["queue_wait_ms"],
0.0,
),
3,
)
base["status_code"] = status_code
base["success"] = success
base["error_category"] = category
@@ -223,29 +289,71 @@ class GatewayRuntime:
if timing["circuit_open"]:
finalize(503, False, "circuit_open")
return JSONResponse(status_code=503, content={"error": {"message": "Upstream temporarily unavailable", "type": "circuit_open"}})
return JSONResponse(
status_code=503,
content={
"error": {
"message": "Upstream temporarily unavailable",
"type": "circuit_open",
}
},
)
if exc is not None:
finalize(502, False, classify_error(0, exc))
return JSONResponse(status_code=502, content={"error": {"message": f"Upstream connection failed: {exc}", "type": "upstream_error"}})
return JSONResponse(
status_code=502,
content={
"error": {
"message": f"Upstream connection failed: {exc}",
"type": "upstream_error",
}
},
)
if resp.status_code != 200:
finalize(resp.status_code, False, classify_error(resp.status_code, None, resp.text))
finalize(
resp.status_code,
False,
classify_error(resp.status_code, None, resp.text),
)
log(f"chat upstream POST -> {resp.status_code}: {resp.text[:300]}")
return JSONResponse(status_code=resp.status_code, content={"error": {"message": resp.text, "type": "upstream_error"}})
return JSONResponse(
status_code=resp.status_code,
content={"error": {"message": resp.text, "type": "upstream_error"}},
)
try:
data = resp.json()
except ValueError:
self.errors += 1
finalize(502, False, "gateway")
log("chat upstream returned 200 but body was not valid JSON")
return JSONResponse(status_code=502, content={"error": {"message": "invalid upstream response", "type": "upstream_error"}})
return JSONResponse(
status_code=502,
content={
"error": {
"message": "invalid upstream response",
"type": "upstream_error",
}
},
)
finalize(200, True, None, data.get("usage"))
log(f"chat POST -> 200 ({timing['upstream_latency_ms']:.0f}ms)")
if stream:
return StreamingResponse(_fake_stream(data, model), media_type="text/event-stream")
return StreamingResponse(
_fake_stream(data, model), media_type="text/event-stream"
)
return JSONResponse(content=data)
async def handle_passthrough(self, method: str, subpath: str, content_type: str, body: bytes,
cfg: dict, owner: tuple, user_agent: str, log=None):
async def handle_passthrough(
self,
method: str,
subpath: str,
content_type: str,
body: bytes,
cfg: dict,
owner: tuple,
user_agent: str,
log=None,
):
log = log or (lambda message: None)
client, sem = self._ensure(cfg)
pricing = pricing_from_cfg(cfg)
@@ -261,18 +369,33 @@ class GatewayRuntime:
if content_type:
headers["Content-Type"] = content_type
resp, exc, timing = await self._send(client, sem, method, url, headers, cfg, log, content=body)
resp, exc, timing = await self._send(
client, sem, method, url, headers, cfg, log, content=body
)
base = {
"owner_kind": owner[0], "owner_id": owner[1], "backend": "chat",
"endpoint": subpath, "model": cfg["gateway_model"], "user_agent": user_agent,
"owner_kind": owner[0],
"owner_id": owner[1],
"backend": "chat",
"endpoint": subpath,
"model": cfg["gateway_model"],
"user_agent": user_agent,
**timing,
}
def finalize(status_code, success, category, usage=None):
base["total_latency_ms"] = round((time.monotonic() - handle_start) * 1000, 3)
base["gateway_overhead_ms"] = round(max(
base["total_latency_ms"] - timing["upstream_latency_ms"] - timing["queue_wait_ms"], 0.0), 3)
base["total_latency_ms"] = round(
(time.monotonic() - handle_start) * 1000, 3
)
base["gateway_overhead_ms"] = round(
max(
base["total_latency_ms"]
- timing["upstream_latency_ms"]
- timing["queue_wait_ms"],
0.0,
),
3,
)
base["status_code"] = status_code
base["success"] = success
base["error_category"] = category
@@ -281,21 +404,50 @@ class GatewayRuntime:
if timing["circuit_open"]:
finalize(503, False, "circuit_open")
return JSONResponse(status_code=503, content={"error": {"message": "Upstream temporarily unavailable", "type": "circuit_open"}})
return JSONResponse(
status_code=503,
content={
"error": {
"message": "Upstream temporarily unavailable",
"type": "circuit_open",
}
},
)
if exc is not None:
finalize(502, False, classify_error(0, exc))
return JSONResponse(status_code=502, content={"error": {"message": f"Upstream connection failed: {exc}", "type": "upstream_error"}})
return JSONResponse(
status_code=502,
content={
"error": {
"message": f"Upstream connection failed: {exc}",
"type": "upstream_error",
}
},
)
usage = None
if resp.status_code < 400 and "application/json" in (resp.headers.get("content-type") or ""):
if resp.status_code < 400 and "application/json" in (
resp.headers.get("content-type") or ""
):
try:
usage = resp.json().get("usage")
except ValueError:
usage = None
finalize(resp.status_code, resp.status_code < 400,
None if resp.status_code < 400 else classify_error(resp.status_code, None, resp.text), usage)
log(f"passthrough {method} {url} -> {resp.status_code} ({timing['upstream_latency_ms']:.0f}ms)")
return Response(content=resp.content, status_code=resp.status_code,
media_type=resp.headers.get("content-type"))
finalize(
resp.status_code,
resp.status_code < 400,
None
if resp.status_code < 400
else classify_error(resp.status_code, None, resp.text),
usage,
)
log(
f"passthrough {method} {url} -> {resp.status_code} ({timing['upstream_latency_ms']:.0f}ms)"
)
return Response(
content=resp.content,
status_code=resp.status_code,
media_type=resp.headers.get("content-type"),
)
def metrics(self) -> dict:
return {
devplacepy/services/openai_gateway/reliability.py
+3 -1
View File
@@ -81,7 +81,9 @@ async def retry_send(
last_exc = exc
if attempts > max_retries:
return None, exc, attempts
log(f"upstream connection failed, retrying ({attempts}/{max_retries}): {exc}")
log(
f"upstream connection failed, retrying ({attempts}/{max_retries}): {exc}"
)
await _backoff(backoff_ms, attempts)
continue
if resp.status_code >= 500 and attempts <= max_retries:
devplacepy/services/openai_gateway/service.py
+280 -89
View File
@@ -36,81 +36,237 @@ class GatewayService(BaseService):
"instances setting."
)
config_fields = [
ConfigField("gateway_upstream_url", "Upstream URL", type="url", default=config.UPSTREAM_URL_DEFAULT,
help="OpenAI-compatible chat-completions endpoint requests are forwarded to.",
group="Upstream"),
ConfigField("gateway_model", "Model", type="str", default=config.MODEL_DEFAULT,
help="Model sent upstream.", group="Upstream"),
ConfigField("gateway_force_model", "Force model", type="bool", default=True,
help="Override the client-requested model with the configured model.", group="Upstream"),
ConfigField("gateway_api_key", "Upstream API key", type="str", default="",
help="The key currently in use; auto-migrated from DEEPSEEK_API_KEY or OPENROUTER_API_KEY on boot. Editable.",
group="Upstream"),
ConfigField("gateway_timeout", "Upstream timeout (seconds)", type="int", default=config.TIMEOUT_DEFAULT, minimum=config.TIMEOUT_MIN,
help="Per-request upstream timeout. Minimum five minutes.", group="Upstream"),
ConfigField("gateway_instances", "Instances (concurrency)", type="int",
default=config.INSTANCES_DEFAULT, minimum=1, maximum=64,
help="Max concurrent upstream forwards per worker (connection pool + semaphore).",
group="Upstream"),
ConfigField("gateway_vision_enabled", "Vision augmentation", type="bool", default=True,
help="Describe image content via the vision model before forwarding.", group="Vision"),
ConfigField("gateway_vision_url", "Vision URL", type="url", default=config.VISION_URL_DEFAULT,
help="OpenAI-compatible endpoint used to describe images.", group="Vision"),
ConfigField("gateway_vision_model", "Vision model", type="str", default=config.VISION_MODEL_DEFAULT,
help="Vision-capable model name.", group="Vision"),
ConfigField("gateway_vision_key", "Vision API key", type="str", default="",
help="The key currently in use; auto-migrated from OPENROUTER_API_KEY on boot. Editable.",
group="Vision"),
ConfigField("gateway_vision_cache_size", "Vision cache size", type="int",
default=config.VISION_CACHE_SIZE_DEFAULT, minimum=0,
help="Image-description LRU cache entries (0 disables caching).", group="Vision"),
ConfigField("gateway_require_auth", "Require authentication", type="bool", default=True,
help="When off, the gateway is open to anyone.", group="Access"),
ConfigField("gateway_allow_admins", "Allow admins", type="bool", default=True,
help="Admin users (API key / Bearer / Basic / session) may call the gateway.", group="Access"),
ConfigField("gateway_allow_users", "Allow users", type="bool", default=True,
help="Any authenticated user may call the gateway with their own API key. "
"Devii operates a signed-in user's account with that user's key, so usage is "
"attributed and limitable per user.", group="Access"),
ConfigField("gateway_access_key", "Static access key", type="password", default="", secret=True,
help="A standalone key that always grants access (sent as X-API-KEY or Bearer).",
group="Access"),
ConfigField("gateway_internal_key", "Internal key", type="str", default="",
help="Auto-generated on boot. DevPlace's own services authenticate to the gateway "
"with this key. Clear it and restart to rotate.",
group="Access"),
ConfigField("gateway_price_cache_hit_per_m", "Chat price cache-hit / 1M ($)", type="float",
default=config.PRICE_CACHE_HIT_PER_M_DEFAULT, minimum=0,
help="Estimates chat cost when the upstream returns no native cost field (DeepSeek).",
group="Pricing"),
ConfigField("gateway_price_cache_miss_per_m", "Chat price cache-miss / 1M ($)", type="float",
default=config.PRICE_CACHE_MISS_PER_M_DEFAULT, minimum=0, group="Pricing"),
ConfigField("gateway_price_output_per_m", "Chat price output / 1M ($)", type="float",
default=config.PRICE_OUTPUT_PER_M_DEFAULT, minimum=0, group="Pricing"),
ConfigField("gateway_vision_price_input_per_m", "Vision price input / 1M ($)", type="float",
default=config.VISION_PRICE_INPUT_PER_M_DEFAULT, minimum=0,
help="Fallback only; used when the vision upstream returns no native cost.",
group="Pricing"),
ConfigField("gateway_vision_price_output_per_m", "Vision price output / 1M ($)", type="float",
default=config.VISION_PRICE_OUTPUT_PER_M_DEFAULT, minimum=0, group="Pricing"),
ConfigField("gateway_max_retries", "Max retries", type="int", default=config.MAX_RETRIES_DEFAULT,
minimum=0, maximum=10,
help="Retry attempts on timeout, connection error, or upstream 5xx.", group="Reliability"),
ConfigField("gateway_retry_backoff_ms", "Retry backoff (ms)", type="int",
default=config.RETRY_BACKOFF_MS_DEFAULT, minimum=0,
help="Linear backoff multiplied by the attempt number.", group="Reliability"),
ConfigField("gateway_circuit_threshold", "Circuit breaker threshold", type="int",
default=config.CIRCUIT_THRESHOLD_DEFAULT, minimum=0,
help="Consecutive upstream failures before the breaker opens (0 disables).", group="Reliability"),
ConfigField("gateway_circuit_cooldown_seconds", "Circuit breaker cooldown (s)", type="int",
default=config.CIRCUIT_COOLDOWN_SECONDS_DEFAULT, minimum=1, group="Reliability"),
ConfigField("gateway_usage_retention_hours", "Usage retention (hours)", type="int",
default=config.USAGE_RETENTION_HOURS_DEFAULT, minimum=1,
help="How long per-call usage rows are kept before pruning.", group="Tracking"),
ConfigField("gateway_model_context_map", "Model context map (JSON)", type="str",
default=json.dumps(config.MODEL_CONTEXT_MAP_DEFAULT),
help="JSON object mapping model name to max context tokens for utilization tracking.",
group="Tracking"),
ConfigField(
"gateway_upstream_url",
"Upstream URL",
type="url",
default=config.UPSTREAM_URL_DEFAULT,
help="OpenAI-compatible chat-completions endpoint requests are forwarded to.",
group="Upstream",
),
ConfigField(
"gateway_model",
"Model",
type="str",
default=config.MODEL_DEFAULT,
help="Model sent upstream.",
group="Upstream",
),
ConfigField(
"gateway_force_model",
"Force model",
type="bool",
default=True,
help="Override the client-requested model with the configured model.",
group="Upstream",
),
ConfigField(
"gateway_api_key",
"Upstream API key",
type="str",
default="",
help="The key currently in use; auto-migrated from DEEPSEEK_API_KEY or OPENROUTER_API_KEY on boot. Editable.",
group="Upstream",
),
ConfigField(
"gateway_timeout",
"Upstream timeout (seconds)",
type="int",
default=config.TIMEOUT_DEFAULT,
minimum=config.TIMEOUT_MIN,
help="Per-request upstream timeout. Minimum five minutes.",
group="Upstream",
),
ConfigField(
"gateway_instances",
"Instances (concurrency)",
type="int",
default=config.INSTANCES_DEFAULT,
minimum=1,
maximum=64,
help="Max concurrent upstream forwards per worker (connection pool + semaphore).",
group="Upstream",
),
ConfigField(
"gateway_vision_enabled",
"Vision augmentation",
type="bool",
default=True,
help="Describe image content via the vision model before forwarding.",
group="Vision",
),
ConfigField(
"gateway_vision_url",
"Vision URL",
type="url",
default=config.VISION_URL_DEFAULT,
help="OpenAI-compatible endpoint used to describe images.",
group="Vision",
),
ConfigField(
"gateway_vision_model",
"Vision model",
type="str",
default=config.VISION_MODEL_DEFAULT,
help="Vision-capable model name.",
group="Vision",
),
ConfigField(
"gateway_vision_key",
"Vision API key",
type="str",
default="",
help="The key currently in use; auto-migrated from OPENROUTER_API_KEY on boot. Editable.",
group="Vision",
),
ConfigField(
"gateway_vision_cache_size",
"Vision cache size",
type="int",
default=config.VISION_CACHE_SIZE_DEFAULT,
minimum=0,
help="Image-description LRU cache entries (0 disables caching).",
group="Vision",
),
ConfigField(
"gateway_require_auth",
"Require authentication",
type="bool",
default=True,
help="When off, the gateway is open to anyone.",
group="Access",
),
ConfigField(
"gateway_allow_admins",
"Allow admins",
type="bool",
default=True,
help="Admin users (API key / Bearer / Basic / session) may call the gateway.",
group="Access",
),
ConfigField(
"gateway_allow_users",
"Allow users",
type="bool",
default=True,
help="Any authenticated user may call the gateway with their own API key. "
"Devii operates a signed-in user's account with that user's key, so usage is "
"attributed and limitable per user.",
group="Access",
),
ConfigField(
"gateway_access_key",
"Static access key",
type="password",
default="",
secret=True,
help="A standalone key that always grants access (sent as X-API-KEY or Bearer).",
group="Access",
),
ConfigField(
"gateway_internal_key",
"Internal key",
type="str",
default="",
help="Auto-generated on boot. DevPlace's own services authenticate to the gateway "
"with this key. Clear it and restart to rotate.",
group="Access",
),
ConfigField(
"gateway_price_cache_hit_per_m",
"Chat price cache-hit / 1M ($)",
type="float",
default=config.PRICE_CACHE_HIT_PER_M_DEFAULT,
minimum=0,
help="Estimates chat cost when the upstream returns no native cost field (DeepSeek).",
group="Pricing",
),
ConfigField(
"gateway_price_cache_miss_per_m",
"Chat price cache-miss / 1M ($)",
type="float",
default=config.PRICE_CACHE_MISS_PER_M_DEFAULT,
minimum=0,
group="Pricing",
),
ConfigField(
"gateway_price_output_per_m",
"Chat price output / 1M ($)",
type="float",
default=config.PRICE_OUTPUT_PER_M_DEFAULT,
minimum=0,
group="Pricing",
),
ConfigField(
"gateway_vision_price_input_per_m",
"Vision price input / 1M ($)",
type="float",
default=config.VISION_PRICE_INPUT_PER_M_DEFAULT,
minimum=0,
help="Fallback only; used when the vision upstream returns no native cost.",
group="Pricing",
),
ConfigField(
"gateway_vision_price_output_per_m",
"Vision price output / 1M ($)",
type="float",
default=config.VISION_PRICE_OUTPUT_PER_M_DEFAULT,
minimum=0,
group="Pricing",
),
ConfigField(
"gateway_max_retries",
"Max retries",
type="int",
default=config.MAX_RETRIES_DEFAULT,
minimum=0,
maximum=10,
help="Retry attempts on timeout, connection error, or upstream 5xx.",
group="Reliability",
),
ConfigField(
"gateway_retry_backoff_ms",
"Retry backoff (ms)",
type="int",
default=config.RETRY_BACKOFF_MS_DEFAULT,
minimum=0,
help="Linear backoff multiplied by the attempt number.",
group="Reliability",
),
ConfigField(
"gateway_circuit_threshold",
"Circuit breaker threshold",
type="int",
default=config.CIRCUIT_THRESHOLD_DEFAULT,
minimum=0,
help="Consecutive upstream failures before the breaker opens (0 disables).",
group="Reliability",
),
ConfigField(
"gateway_circuit_cooldown_seconds",
"Circuit breaker cooldown (s)",
type="int",
default=config.CIRCUIT_COOLDOWN_SECONDS_DEFAULT,
minimum=1,
group="Reliability",
),
ConfigField(
"gateway_usage_retention_hours",
"Usage retention (hours)",
type="int",
default=config.USAGE_RETENTION_HOURS_DEFAULT,
minimum=1,
help="How long per-call usage rows are kept before pruning.",
group="Tracking",
),
ConfigField(
"gateway_model_context_map",
"Model context map (JSON)",
type="str",
default=json.dumps(config.MODEL_CONTEXT_MAP_DEFAULT),
help="JSON object mapping model name to max context tokens for utilization tracking.",
group="Tracking",
),
]
def __init__(self):
@@ -124,11 +280,14 @@ class GatewayService(BaseService):
def effective_config(self) -> dict:
cfg = self.get_config()
cfg["gateway_api_key"] = (cfg["gateway_api_key"]
or os.environ.get("DEEPSEEK_API_KEY", "")
or os.environ.get("OPENROUTER_API_KEY", ""))
cfg["gateway_vision_key"] = (cfg["gateway_vision_key"]
or os.environ.get("OPENROUTER_API_KEY", ""))
cfg["gateway_api_key"] = (
cfg["gateway_api_key"]
or os.environ.get("DEEPSEEK_API_KEY", "")
or os.environ.get("OPENROUTER_API_KEY", "")
)
cfg["gateway_vision_key"] = cfg["gateway_vision_key"] or os.environ.get(
"OPENROUTER_API_KEY", ""
)
return cfg
def authorize(self, request: Request) -> bool:
@@ -153,9 +312,17 @@ class GatewayService(BaseService):
def resolve_owner(self, request: Request) -> tuple:
cfg = self.get_config()
presented = _presented_key(request)
if presented and cfg["gateway_internal_key"] and presented == cfg["gateway_internal_key"]:
if (
presented
and cfg["gateway_internal_key"]
and presented == cfg["gateway_internal_key"]
):
return ("internal", "devii")
if presented and cfg["gateway_access_key"] and presented == cfg["gateway_access_key"]:
if (
presented
and cfg["gateway_access_key"]
and presented == cfg["gateway_access_key"]
):
return ("key", "access")
user = get_current_user(request)
if user:
@@ -185,7 +352,16 @@ class GatewayService(BaseService):
return await runtime.handle_chat(body, cfg, owner, user_agent, self.log)
body = await request.body()
content_type = request.headers.get("content-type", "")
return await runtime.handle_passthrough(request.method, subpath, content_type, body, cfg, owner, user_agent, self.log)
return await runtime.handle_passthrough(
request.method,
subpath,
content_type,
body,
cfg,
owner,
user_agent,
self.log,
)
async def run_once(self) -> None:
if not self.is_enabled():
@@ -193,10 +369,14 @@ class GatewayService(BaseService):
runtime = self.runtime()
runtime._ensure(self.effective_config())
runtime._ledger.sample_concurrency(runtime.in_flight)
retention = get_int_setting("gateway_usage_retention_hours", config.USAGE_RETENTION_HOURS_DEFAULT)
retention = get_int_setting(
"gateway_usage_retention_hours", config.USAGE_RETENTION_HOURS_DEFAULT
)
ledger_removed, samples_removed = runtime._ledger.prune(retention)
if ledger_removed or samples_removed:
self.log(f"Pruned {ledger_removed} usage rows and {samples_removed} concurrency samples")
self.log(
f"Pruned {ledger_removed} usage rows and {samples_removed} concurrency samples"
)
async def on_disable(self) -> None:
if self._runtime is not None:
@@ -204,10 +384,21 @@ class GatewayService(BaseService):
def collect_metrics(self) -> dict:
cfg = self.get_config()
m = self._runtime.metrics() if self._runtime is not None else {
"requests": 0, "errors": 0, "in_flight": 0, "peak_in_flight": 0, "vision_calls": 0,
"last_status": 0, "last_latency_ms": 0, "pool": 0, "circuit_open": False,
}
m = (
self._runtime.metrics()
if self._runtime is not None
else {
"requests": 0,
"errors": 0,
"in_flight": 0,
"peak_in_flight": 0,
"vision_calls": 0,
"last_status": 0,
"last_latency_ms": 0,
"pool": 0,
"circuit_open": False,
}
)
s = summary_metrics()
stats = [
{"label": "Requests (lifetime)", "value": m["requests"]},
devplacepy/services/openai_gateway/usage.py
+78 -24
View File
@@ -39,11 +39,31 @@ class Pricing:
def pricing_from_cfg(cfg: dict) -> Pricing:
return Pricing(
chat_cache_hit_per_m=float(cfg.get("gateway_price_cache_hit_per_m", config.PRICE_CACHE_HIT_PER_M_DEFAULT)),
chat_cache_miss_per_m=float(cfg.get("gateway_price_cache_miss_per_m", config.PRICE_CACHE_MISS_PER_M_DEFAULT)),
chat_output_per_m=float(cfg.get("gateway_price_output_per_m", config.PRICE_OUTPUT_PER_M_DEFAULT)),
vision_input_per_m=float(cfg.get("gateway_vision_price_input_per_m", config.VISION_PRICE_INPUT_PER_M_DEFAULT)),
vision_output_per_m=float(cfg.get("gateway_vision_price_output_per_m", config.VISION_PRICE_OUTPUT_PER_M_DEFAULT)),
chat_cache_hit_per_m=float(
cfg.get(
"gateway_price_cache_hit_per_m", config.PRICE_CACHE_HIT_PER_M_DEFAULT
)
),
chat_cache_miss_per_m=float(
cfg.get(
"gateway_price_cache_miss_per_m", config.PRICE_CACHE_MISS_PER_M_DEFAULT
)
),
chat_output_per_m=float(
cfg.get("gateway_price_output_per_m", config.PRICE_OUTPUT_PER_M_DEFAULT)
),
vision_input_per_m=float(
cfg.get(
"gateway_vision_price_input_per_m",
config.VISION_PRICE_INPUT_PER_M_DEFAULT,
)
),
vision_output_per_m=float(
cfg.get(
"gateway_vision_price_output_per_m",
config.VISION_PRICE_OUTPUT_PER_M_DEFAULT,
)
),
)
@@ -90,7 +110,9 @@ def normalize_usage(usage: Optional[dict]) -> dict:
}
def compute_cost(usage: dict, norm: dict, pricing: Pricing, backend: str) -> tuple[float, float, float, bool]:
def compute_cost(
usage: dict, norm: dict, pricing: Pricing, backend: str
) -> tuple[float, float, float, bool]:
native = usage.get("cost") if isinstance(usage, dict) else None
if isinstance(native, (int, float)) and not isinstance(native, bool):
total = float(native)
@@ -101,13 +123,17 @@ def compute_cost(usage: dict, norm: dict, pricing: Pricing, backend: str) -> tup
input_cost = norm["prompt"] / PER_MILLION * pricing.vision_input_per_m
output_cost = norm["completion"] / PER_MILLION * pricing.vision_output_per_m
return input_cost + output_cost, input_cost, output_cost, False
input_cost = (norm["cache_hit"] / PER_MILLION * pricing.chat_cache_hit_per_m
+ norm["cache_miss"] / PER_MILLION * pricing.chat_cache_miss_per_m)
input_cost = (
norm["cache_hit"] / PER_MILLION * pricing.chat_cache_hit_per_m
+ norm["cache_miss"] / PER_MILLION * pricing.chat_cache_miss_per_m
)
output_cost = norm["completion"] / PER_MILLION * pricing.chat_output_per_m
return input_cost + output_cost, input_cost, output_cost, False
def context_utilization(total_tokens: int, model: str, context_map: dict) -> tuple[Optional[int], Optional[float]]:
def context_utilization(
total_tokens: int, model: str, context_map: dict
) -> tuple[Optional[int], Optional[float]]:
window = context_map.get(model)
if not window or window <= 0:
return None, None
@@ -116,8 +142,14 @@ def context_utilization(total_tokens: int, model: str, context_map: dict) -> tup
def extract_params(body: Any) -> dict:
if not isinstance(body, dict):
return {"requested_model": "", "stream_requested": False, "temperature": None,
"top_p": None, "max_tokens": None, "has_tools": False}
return {
"requested_model": "",
"stream_requested": False,
"temperature": None,
"top_p": None,
"max_tokens": None,
"has_tools": False,
}
temperature = body.get("temperature")
top_p = body.get("top_p")
max_tokens = body.get("max_tokens")
@@ -126,14 +158,22 @@ def extract_params(body: Any) -> dict:
return {
"requested_model": body.get("model") or "",
"stream_requested": bool(body.get("stream")),
"temperature": float(temperature) if isinstance(temperature, (int, float)) and not isinstance(temperature, bool) else None,
"top_p": float(top_p) if isinstance(top_p, (int, float)) and not isinstance(top_p, bool) else None,
"max_tokens": int(max_tokens) if isinstance(max_tokens, (int, float)) and not isinstance(max_tokens, bool) else None,
"temperature": float(temperature)
if isinstance(temperature, (int, float)) and not isinstance(temperature, bool)
else None,
"top_p": float(top_p)
if isinstance(top_p, (int, float)) and not isinstance(top_p, bool)
else None,
"max_tokens": int(max_tokens)
if isinstance(max_tokens, (int, float)) and not isinstance(max_tokens, bool)
else None,
"has_tools": bool(body.get("tools") or body.get("functions")),
}
def classify_error(status_code: int, exc: Optional[Exception] = None, message: str = "") -> str:
def classify_error(
status_code: int, exc: Optional[Exception] = None, message: str = ""
) -> str:
if exc is not None:
if isinstance(exc, httpx.TimeoutException):
return "timeout"
@@ -159,11 +199,19 @@ class GatewayUsageLedger:
try:
usage = raw.get("usage") or {}
norm = normalize_usage(usage)
cost_usd, input_cost, output_cost, native = compute_cost(usage, norm, pricing, raw["backend"])
window, util = context_utilization(norm["total"], raw.get("model") or "", context_map)
cost_usd, input_cost, output_cost, native = compute_cost(
usage, norm, pricing, raw["backend"]
)
window, util = context_utilization(
norm["total"], raw.get("model") or "", context_map
)
upstream_ms = float(raw.get("upstream_latency_ms") or 0)
completion = norm["completion"]
tps = completion / (upstream_ms / 1000.0) if upstream_ms > 0 and completion else 0.0
tps = (
completion / (upstream_ms / 1000.0)
if upstream_ms > 0 and completion
else 0.0
)
row = {
"created_at": _iso(_now()),
"owner_kind": raw.get("owner_kind") or "unknown",
@@ -209,10 +257,12 @@ class GatewayUsageLedger:
def sample_concurrency(self, in_flight: int) -> None:
try:
get_table(GATEWAY_CONCURRENCY).insert({
"created_at": _iso(_now()),
"in_flight": int(in_flight),
})
get_table(GATEWAY_CONCURRENCY).insert(
{
"created_at": _iso(_now()),
"in_flight": int(in_flight),
}
)
except Exception as exc:
logger.warning("gateway concurrency sample failed: %s", exc)
@@ -221,7 +271,11 @@ class GatewayUsageLedger:
ledger_removed = 0
samples_removed = 0
if GATEWAY_LEDGER in db.tables:
ledger_removed = int(get_table(GATEWAY_LEDGER).delete(created_at={"<": cutoff}))
ledger_removed = int(
get_table(GATEWAY_LEDGER).delete(created_at={"<": cutoff})
)
if GATEWAY_CONCURRENCY in db.tables:
samples_removed = int(get_table(GATEWAY_CONCURRENCY).delete(created_at={"<": cutoff}))
samples_removed = int(
get_table(GATEWAY_CONCURRENCY).delete(created_at={"<": cutoff})
)
return ledger_removed, samples_removed
devplacepy/services/openai_gateway/vision.py
+67 -25
View File
@@ -42,8 +42,7 @@ def has_vision_blocks(content: Any) -> bool:
if not isinstance(content, list):
return False
return any(
isinstance(b, dict) and b.get("type") in ("image_url", "image")
for b in content
isinstance(b, dict) and b.get("type") in ("image_url", "image") for b in content
)
@@ -73,13 +72,27 @@ def _format_vision_block(descriptions: list) -> str:
if len(descriptions) == 1:
return f"[Image seen by vision model:\n{descriptions[0]}\n]"
parts = [f"Image {i}:\n{d}" for i, d in enumerate(descriptions, 1)]
return f"[{len(descriptions)} images seen by vision model:\n" + "\n\n".join(parts) + "\n]"
return (
f"[{len(descriptions)} images seen by vision model:\n"
+ "\n\n".join(parts)
+ "\n]"
)
class VisionAugmenter:
def __init__(self, vision_url: str, vision_model: str, vision_key: str,
cache: VisionCache, referer: str = "", title: str = "",
ledger=None, owner: tuple = ("unknown", "unknown"), pricing=None, context_map=None):
def __init__(
self,
vision_url: str,
vision_model: str,
vision_key: str,
cache: VisionCache,
referer: str = "",
title: str = "",
ledger=None,
owner: tuple = ("unknown", "unknown"),
pricing=None,
context_map=None,
):
self.vision_url = vision_url
self.vision_model = vision_model
self.vision_key = vision_key
@@ -95,27 +108,40 @@ class VisionAugmenter:
def _record(self, latency_ms, status_code, success, category, usage):
if self.ledger is None or self.pricing is None:
return
self.ledger.record({
"owner_kind": self.owner[0], "owner_id": self.owner[1], "backend": "vision",
"endpoint": "chat/completions", "model": self.vision_model,
"requested_model": self.vision_model, "temperature": 0.2,
"upstream_latency_ms": latency_ms, "total_latency_ms": latency_ms,
"status_code": status_code, "success": success, "error_category": category,
"usage": usage,
}, self.pricing, self.context_map)
self.ledger.record(
{
"owner_kind": self.owner[0],
"owner_id": self.owner[1],
"backend": "vision",
"endpoint": "chat/completions",
"model": self.vision_model,
"requested_model": self.vision_model,
"temperature": 0.2,
"upstream_latency_ms": latency_ms,
"total_latency_ms": latency_ms,
"status_code": status_code,
"success": success,
"error_category": category,
"usage": usage,
},
self.pricing,
self.context_map,
)
async def _describe_one(self, client: httpx.AsyncClient, image_block: dict) -> str:
if not self.vision_key:
return "[vision unavailable: vision API key not configured]"
payload = {
"model": self.vision_model,
"messages": [{
"role": "user",
"content": [
{"type": "text", "text": VISION_INSTRUCTION},
image_block,
],
}],
"messages": [
{
"role": "user",
"content": [
{"type": "text", "text": VISION_INSTRUCTION},
image_block,
],
}
],
"temperature": 0.2,
"stream": False,
}
@@ -132,12 +158,24 @@ class VisionAugmenter:
resp = await client.post(self.vision_url, json=payload, headers=headers)
except httpx.RequestError as e:
logger.warning("vision connection failed: %s", e)
self._record((time.monotonic() - start) * 1000, 502, False, classify_error(0, e), None)
self._record(
(time.monotonic() - start) * 1000,
502,
False,
classify_error(0, e),
None,
)
return f"[vision call failed: {e}]"
latency_ms = (time.monotonic() - start) * 1000
if resp.status_code != 200:
logger.warning("vision %s: %s", resp.status_code, resp.text[:200])
self._record(latency_ms, resp.status_code, False, classify_error(resp.status_code, None, resp.text), None)
self._record(
latency_ms,
resp.status_code,
False,
classify_error(resp.status_code, None, resp.text),
None,
)
return f"[vision failed: HTTP {resp.status_code}]"
try:
data = resp.json()
@@ -154,7 +192,9 @@ class VisionAugmenter:
miss = [i for i, d in enumerate(descriptions) if d is None]
if miss:
self.calls += len(miss)
results = await asyncio.gather(*[self._describe_one(client, images[i]) for i in miss])
results = await asyncio.gather(
*[self._describe_one(client, images[i]) for i in miss]
)
for idx, result in zip(miss, results):
descriptions[idx] = result
if not result.startswith("[vision"):
@@ -162,7 +202,9 @@ class VisionAugmenter:
return [d or "" for d in descriptions]
async def augment_messages(self, client: httpx.AsyncClient, messages: list) -> list:
if not any(has_vision_blocks(m.get("content")) for m in messages if isinstance(m, dict)):
if not any(
has_vision_blocks(m.get("content")) for m in messages if isinstance(m, dict)
):
return messages
out: list = []
for m in messages: