retoor/devplacepy
1
0
Fork 1
Code Issues 44 Pull Requests 4 Actions Packages Projects Releases Wiki Activity

fix: normalize unicode escape sequences and reformat multi-line expressions across codebase

Browse Source
This commit is contained in:
retoor
2026-06-09 16:48:08 +00:00
parent 66dfda88bc
commit c4f2937415
175 changed files with 12660 additions and 4175 deletions
Download Patch File Download Diff File Expand all files Collapse all files
devplacepy/services/openai_gateway/gateway.py
+207 -55
View File
@@ -36,24 +36,30 @@ def _fake_stream(data: dict, model: str):
reasoning_content = msg.get("reasoning_content") or ""
def _chunk(delta: dict, finish: Optional[str] = None) -> str:
return "data: " + json.dumps({
"id": chunk_id,
"object": "chat.completion.chunk",
"created": created,
"model": out_model,
"choices": [{"index": 0, "delta": delta, "finish_reason": finish}],
}) + "\n\n"
return (
"data: "
+ json.dumps(
{
"id": chunk_id,
"object": "chat.completion.chunk",
"created": created,
"model": out_model,
"choices": [{"index": 0, "delta": delta, "finish_reason": finish}],
}
)
+ "\n\n"
)
async def gen():
yield _chunk({"role": "assistant"})
if reasoning_content:
for i in range(0, len(reasoning_content), 50):
yield _chunk({"reasoning_content": reasoning_content[i:i + 50]})
yield _chunk({"reasoning_content": reasoning_content[i : i + 50]})
if tool_calls:
yield _chunk({"tool_calls": tool_calls})
elif content:
for i in range(0, len(content), 50):
yield _chunk({"content": content[i:i + 50]})
yield _chunk({"content": content[i : i + 50]})
yield _chunk({}, finish="tool_calls" if tool_calls else "stop")
yield "data: [DONE]\n\n"
@@ -66,7 +72,9 @@ def _connect_tracer(holder: dict):
async def trace(name: str, info: dict) -> None:
if name.endswith("connect_tcp.started") or name.endswith("start_tls.started"):
started[name] = time.monotonic()
elif name.endswith("connect_tcp.complete") or name.endswith("start_tls.complete"):
elif name.endswith("connect_tcp.complete") or name.endswith(
"start_tls.complete"
):
begin = started.get(name.replace(".complete", ".started"))
if begin is not None:
holder["ms"] += (time.monotonic() - begin) * 1000
@@ -83,7 +91,9 @@ class GatewayRuntime:
self._vision_cache: Optional[VisionCache] = None
self._vision_cache_size = -1
self._ledger = GatewayUsageLedger()
self._breaker = CircuitBreaker(config.CIRCUIT_THRESHOLD_DEFAULT, config.CIRCUIT_COOLDOWN_SECONDS_DEFAULT)
self._breaker = CircuitBreaker(
config.CIRCUIT_THRESHOLD_DEFAULT, config.CIRCUIT_COOLDOWN_SECONDS_DEFAULT
)
self.requests = 0
self.errors = 0
self.in_flight = 0
@@ -95,9 +105,15 @@ class GatewayRuntime:
def _ensure(self, cfg: dict):
instances = max(1, cfg["gateway_instances"])
timeout = max(1, cfg["gateway_timeout"])
if self._client is None or instances != self._instances or timeout != self._timeout:
if (
self._client is None
or instances != self._instances
or timeout != self._timeout
):
old = self._client
limits = httpx.Limits(max_connections=instances, max_keepalive_connections=instances)
limits = httpx.Limits(
max_connections=instances, max_keepalive_connections=instances
)
self._client = httpx.AsyncClient(timeout=float(timeout), limits=limits)
self._sem = asyncio.Semaphore(instances)
self._instances = instances
@@ -108,7 +124,9 @@ class GatewayRuntime:
if self._vision_cache is None or size != self._vision_cache_size:
self._vision_cache = VisionCache(size)
self._vision_cache_size = size
self._breaker.configure(cfg["gateway_circuit_threshold"], cfg["gateway_circuit_cooldown_seconds"])
self._breaker.configure(
cfg["gateway_circuit_threshold"], cfg["gateway_circuit_cooldown_seconds"]
)
return self._client, self._sem
async def aclose(self) -> None:
@@ -117,9 +135,17 @@ class GatewayRuntime:
self._client = None
self._instances = 0
async def _send(self, client, sem, method, url, headers, cfg, log, json_body=None, content=None):
timing = {"queue_wait_ms": 0.0, "upstream_latency_ms": 0.0, "connect_ms": 0.0,
"retries_attempted": 0, "retry_succeeded": False, "circuit_open": False}
async def _send(
self, client, sem, method, url, headers, cfg, log, json_body=None, content=None
):
timing = {
"queue_wait_ms": 0.0,
"upstream_latency_ms": 0.0,
"connect_ms": 0.0,
"retries_attempted": 0,
"retry_succeeded": False,
"circuit_open": False,
}
if not self._breaker.allow():
timing["circuit_open"] = True
log("circuit breaker open, rejecting upstream call")
@@ -135,18 +161,27 @@ class GatewayRuntime:
exc = None
try:
async with sem:
timing["queue_wait_ms"] = round((time.monotonic() - wait_start) * 1000, 3)
timing["queue_wait_ms"] = round(
(time.monotonic() - wait_start) * 1000, 3
)
async def do_call():
request = client.build_request(method, url, headers=headers,
json=json_body, content=content)
request = client.build_request(
method, url, headers=headers, json=json_body, content=content
)
request.extensions["trace"] = _connect_tracer(connect_holder)
return await client.send(request)
send_start = time.monotonic()
resp, exc, attempts = await retry_send(
do_call, cfg["gateway_max_retries"], cfg["gateway_retry_backoff_ms"], log)
timing["upstream_latency_ms"] = round((time.monotonic() - send_start) * 1000, 3)
do_call,
cfg["gateway_max_retries"],
cfg["gateway_retry_backoff_ms"],
log,
)
timing["upstream_latency_ms"] = round(
(time.monotonic() - send_start) * 1000, 3
)
finally:
self.in_flight -= 1
timing["connect_ms"] = round(connect_holder["ms"], 3)
@@ -166,7 +201,9 @@ class GatewayRuntime:
timing["retry_succeeded"] = attempts > 1
return resp, None, timing
async def handle_chat(self, body: dict, cfg: dict, owner: tuple, user_agent: str, log=None):
async def handle_chat(
self, body: dict, cfg: dict, owner: tuple, user_agent: str, log=None
):
log = log or (lambda message: None)
client, sem = self._ensure(cfg)
pricing = pricing_from_cfg(cfg)
@@ -177,8 +214,13 @@ class GatewayRuntime:
if cfg["gateway_vision_enabled"]:
augmenter = VisionAugmenter(
cfg["gateway_vision_url"], cfg["gateway_vision_model"], cfg["gateway_vision_key"],
self._vision_cache, ledger=self._ledger, owner=owner, pricing=pricing,
cfg["gateway_vision_url"],
cfg["gateway_vision_model"],
cfg["gateway_vision_key"],
self._vision_cache,
ledger=self._ledger,
owner=owner,
pricing=pricing,
context_map=context_map,
)
messages = await augmenter.augment_messages(client, messages)
@@ -200,21 +242,45 @@ class GatewayRuntime:
if cfg["gateway_api_key"]:
headers["Authorization"] = f"Bearer {cfg['gateway_api_key']}"
else:
log("No upstream API key configured (gateway_api_key / DEEPSEEK_API_KEY / OPENROUTER_API_KEY); upstream will likely reject the request")
log(
"No upstream API key configured (gateway_api_key / DEEPSEEK_API_KEY / OPENROUTER_API_KEY); upstream will likely reject the request"
)
resp, exc, timing = await self._send(
client, sem, "POST", cfg["gateway_upstream_url"], headers, cfg, log, json_body=payload)
client,
sem,
"POST",
cfg["gateway_upstream_url"],
headers,
cfg,
log,
json_body=payload,
)
base = {
"owner_kind": owner[0], "owner_id": owner[1], "backend": "chat",
"endpoint": "chat/completions", "model": model, "user_agent": user_agent,
**params, **timing,
"owner_kind": owner[0],
"owner_id": owner[1],
"backend": "chat",
"endpoint": "chat/completions",
"model": model,
"user_agent": user_agent,
**params,
**timing,
}
def finalize(status_code, success, category, usage=None):
base["total_latency_ms"] = round((time.monotonic() - handle_start) * 1000, 3)
base["gateway_overhead_ms"] = round(max(
base["total_latency_ms"] - timing["upstream_latency_ms"] - timing["queue_wait_ms"], 0.0), 3)
base["total_latency_ms"] = round(
(time.monotonic() - handle_start) * 1000, 3
)
base["gateway_overhead_ms"] = round(
max(
base["total_latency_ms"]
- timing["upstream_latency_ms"]
- timing["queue_wait_ms"],
0.0,
),
3,
)
base["status_code"] = status_code
base["success"] = success
base["error_category"] = category
@@ -223,29 +289,71 @@ class GatewayRuntime:
if timing["circuit_open"]:
finalize(503, False, "circuit_open")
return JSONResponse(status_code=503, content={"error": {"message": "Upstream temporarily unavailable", "type": "circuit_open"}})
return JSONResponse(
status_code=503,
content={
"error": {
"message": "Upstream temporarily unavailable",
"type": "circuit_open",
}
},
)
if exc is not None:
finalize(502, False, classify_error(0, exc))
return JSONResponse(status_code=502, content={"error": {"message": f"Upstream connection failed: {exc}", "type": "upstream_error"}})
return JSONResponse(
status_code=502,
content={
"error": {
"message": f"Upstream connection failed: {exc}",
"type": "upstream_error",
}
},
)
if resp.status_code != 200:
finalize(resp.status_code, False, classify_error(resp.status_code, None, resp.text))
finalize(
resp.status_code,
False,
classify_error(resp.status_code, None, resp.text),
)
log(f"chat upstream POST -> {resp.status_code}: {resp.text[:300]}")
return JSONResponse(status_code=resp.status_code, content={"error": {"message": resp.text, "type": "upstream_error"}})
return JSONResponse(
status_code=resp.status_code,
content={"error": {"message": resp.text, "type": "upstream_error"}},
)
try:
data = resp.json()
except ValueError:
self.errors += 1
finalize(502, False, "gateway")
log("chat upstream returned 200 but body was not valid JSON")
return JSONResponse(status_code=502, content={"error": {"message": "invalid upstream response", "type": "upstream_error"}})
return JSONResponse(
status_code=502,
content={
"error": {
"message": "invalid upstream response",
"type": "upstream_error",
}
},
)
finalize(200, True, None, data.get("usage"))
log(f"chat POST -> 200 ({timing['upstream_latency_ms']:.0f}ms)")
if stream:
return StreamingResponse(_fake_stream(data, model), media_type="text/event-stream")
return StreamingResponse(
_fake_stream(data, model), media_type="text/event-stream"
)
return JSONResponse(content=data)
async def handle_passthrough(self, method: str, subpath: str, content_type: str, body: bytes,
cfg: dict, owner: tuple, user_agent: str, log=None):
async def handle_passthrough(
self,
method: str,
subpath: str,
content_type: str,
body: bytes,
cfg: dict,
owner: tuple,
user_agent: str,
log=None,
):
log = log or (lambda message: None)
client, sem = self._ensure(cfg)
pricing = pricing_from_cfg(cfg)
@@ -261,18 +369,33 @@ class GatewayRuntime:
if content_type:
headers["Content-Type"] = content_type
resp, exc, timing = await self._send(client, sem, method, url, headers, cfg, log, content=body)
resp, exc, timing = await self._send(
client, sem, method, url, headers, cfg, log, content=body
)
base = {
"owner_kind": owner[0], "owner_id": owner[1], "backend": "chat",
"endpoint": subpath, "model": cfg["gateway_model"], "user_agent": user_agent,
"owner_kind": owner[0],
"owner_id": owner[1],
"backend": "chat",
"endpoint": subpath,
"model": cfg["gateway_model"],
"user_agent": user_agent,
**timing,
}
def finalize(status_code, success, category, usage=None):
base["total_latency_ms"] = round((time.monotonic() - handle_start) * 1000, 3)
base["gateway_overhead_ms"] = round(max(
base["total_latency_ms"] - timing["upstream_latency_ms"] - timing["queue_wait_ms"], 0.0), 3)
base["total_latency_ms"] = round(
(time.monotonic() - handle_start) * 1000, 3
)
base["gateway_overhead_ms"] = round(
max(
base["total_latency_ms"]
- timing["upstream_latency_ms"]
- timing["queue_wait_ms"],
0.0,
),
3,
)
base["status_code"] = status_code
base["success"] = success
base["error_category"] = category
@@ -281,21 +404,50 @@ class GatewayRuntime:
if timing["circuit_open"]:
finalize(503, False, "circuit_open")
return JSONResponse(status_code=503, content={"error": {"message": "Upstream temporarily unavailable", "type": "circuit_open"}})
return JSONResponse(
status_code=503,
content={
"error": {
"message": "Upstream temporarily unavailable",
"type": "circuit_open",
}
},
)
if exc is not None:
finalize(502, False, classify_error(0, exc))
return JSONResponse(status_code=502, content={"error": {"message": f"Upstream connection failed: {exc}", "type": "upstream_error"}})
return JSONResponse(
status_code=502,
content={
"error": {
"message": f"Upstream connection failed: {exc}",
"type": "upstream_error",
}
},
)
usage = None
if resp.status_code < 400 and "application/json" in (resp.headers.get("content-type") or ""):
if resp.status_code < 400 and "application/json" in (
resp.headers.get("content-type") or ""
):
try:
usage = resp.json().get("usage")
except ValueError:
usage = None
finalize(resp.status_code, resp.status_code < 400,
None if resp.status_code < 400 else classify_error(resp.status_code, None, resp.text), usage)
log(f"passthrough {method} {url} -> {resp.status_code} ({timing['upstream_latency_ms']:.0f}ms)")
return Response(content=resp.content, status_code=resp.status_code,
media_type=resp.headers.get("content-type"))
finalize(
resp.status_code,
resp.status_code < 400,
None
if resp.status_code < 400
else classify_error(resp.status_code, None, resp.text),
usage,
)
log(
f"passthrough {method} {url} -> {resp.status_code} ({timing['upstream_latency_ms']:.0f}ms)"
)
return Response(
content=resp.content,
status_code=resp.status_code,
media_type=resp.headers.get("content-type"),
)
def metrics(self) -> dict:
return {