forked from retoor/devplacepy
feat: add api key auth, devii agent, openai gateway, and admin service management
This commit introduces a comprehensive set of new features including API key authentication with CLI management commands (get, reset, backfill), a Devii agentic assistant with WebSocket terminal and session bootstrap, an OpenAI-compatible LLM gateway service, and an admin service management panel. It also adds Playwright browser automation for bot support, configures internal gateway URLs, refactors content editing/deletion to support JSON API responses, and updates documentation across AGENTS.md, README.md, and the developer docs site.
This commit is contained in:
@@ -0,0 +1,311 @@
|
||||
import asyncio
|
||||
import json
|
||||
import logging
|
||||
import time
|
||||
import uuid
|
||||
from typing import Optional
|
||||
|
||||
import httpx
|
||||
from fastapi.responses import JSONResponse, Response, StreamingResponse
|
||||
|
||||
from devplacepy.services.openai_gateway import config
|
||||
from devplacepy.services.openai_gateway.reliability import CircuitBreaker, retry_send
|
||||
from devplacepy.services.openai_gateway.usage import (
|
||||
GatewayUsageLedger,
|
||||
classify_error,
|
||||
extract_params,
|
||||
parse_context_map,
|
||||
pricing_from_cfg,
|
||||
)
|
||||
from devplacepy.services.openai_gateway.vision import VisionAugmenter, VisionCache
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _fake_stream(data: dict, model: str):
|
||||
chunk_id = data.get("id", f"chatcmpl-{uuid.uuid4().hex[:12]}")
|
||||
created = data.get("created", int(time.time()))
|
||||
out_model = data.get("model", model)
|
||||
|
||||
try:
|
||||
msg = data["choices"][0]["message"]
|
||||
except (KeyError, IndexError):
|
||||
msg = {"content": ""}
|
||||
tool_calls = msg.get("tool_calls")
|
||||
content = msg.get("content") or ""
|
||||
reasoning_content = msg.get("reasoning_content") or ""
|
||||
|
||||
def _chunk(delta: dict, finish: Optional[str] = None) -> str:
|
||||
return "data: " + json.dumps({
|
||||
"id": chunk_id,
|
||||
"object": "chat.completion.chunk",
|
||||
"created": created,
|
||||
"model": out_model,
|
||||
"choices": [{"index": 0, "delta": delta, "finish_reason": finish}],
|
||||
}) + "\n\n"
|
||||
|
||||
async def gen():
|
||||
yield _chunk({"role": "assistant"})
|
||||
if reasoning_content:
|
||||
for i in range(0, len(reasoning_content), 50):
|
||||
yield _chunk({"reasoning_content": reasoning_content[i:i + 50]})
|
||||
if tool_calls:
|
||||
yield _chunk({"tool_calls": tool_calls})
|
||||
elif content:
|
||||
for i in range(0, len(content), 50):
|
||||
yield _chunk({"content": content[i:i + 50]})
|
||||
yield _chunk({}, finish="tool_calls" if tool_calls else "stop")
|
||||
yield "data: [DONE]\n\n"
|
||||
|
||||
return gen()
|
||||
|
||||
|
||||
def _connect_tracer(holder: dict):
|
||||
started: dict = {}
|
||||
|
||||
async def trace(name: str, info: dict) -> None:
|
||||
if name.endswith("connect_tcp.started") or name.endswith("start_tls.started"):
|
||||
started[name] = time.monotonic()
|
||||
elif name.endswith("connect_tcp.complete") or name.endswith("start_tls.complete"):
|
||||
begin = started.get(name.replace(".complete", ".started"))
|
||||
if begin is not None:
|
||||
holder["ms"] += (time.monotonic() - begin) * 1000
|
||||
|
||||
return trace
|
||||
|
||||
|
||||
class GatewayRuntime:
|
||||
def __init__(self):
|
||||
self._client: Optional[httpx.AsyncClient] = None
|
||||
self._sem: Optional[asyncio.Semaphore] = None
|
||||
self._instances = 0
|
||||
self._timeout = 0
|
||||
self._vision_cache: Optional[VisionCache] = None
|
||||
self._vision_cache_size = -1
|
||||
self._ledger = GatewayUsageLedger()
|
||||
self._breaker = CircuitBreaker(config.CIRCUIT_THRESHOLD_DEFAULT, config.CIRCUIT_COOLDOWN_SECONDS_DEFAULT)
|
||||
self.requests = 0
|
||||
self.errors = 0
|
||||
self.in_flight = 0
|
||||
self.peak_in_flight = 0
|
||||
self.vision_calls = 0
|
||||
self.last_status = 0
|
||||
self.last_latency_ms = 0
|
||||
|
||||
def _ensure(self, cfg: dict):
|
||||
instances = max(1, cfg["gateway_instances"])
|
||||
timeout = max(1, cfg["gateway_timeout"])
|
||||
if self._client is None or instances != self._instances or timeout != self._timeout:
|
||||
old = self._client
|
||||
limits = httpx.Limits(max_connections=instances, max_keepalive_connections=instances)
|
||||
self._client = httpx.AsyncClient(timeout=float(timeout), limits=limits)
|
||||
self._sem = asyncio.Semaphore(instances)
|
||||
self._instances = instances
|
||||
self._timeout = timeout
|
||||
if old is not None:
|
||||
asyncio.create_task(old.aclose())
|
||||
size = cfg["gateway_vision_cache_size"]
|
||||
if self._vision_cache is None or size != self._vision_cache_size:
|
||||
self._vision_cache = VisionCache(size)
|
||||
self._vision_cache_size = size
|
||||
self._breaker.configure(cfg["gateway_circuit_threshold"], cfg["gateway_circuit_cooldown_seconds"])
|
||||
return self._client, self._sem
|
||||
|
||||
async def aclose(self) -> None:
|
||||
if self._client is not None:
|
||||
await self._client.aclose()
|
||||
self._client = None
|
||||
self._instances = 0
|
||||
|
||||
async def _send(self, client, sem, method, url, headers, cfg, log, json_body=None, content=None):
|
||||
timing = {"queue_wait_ms": 0.0, "upstream_latency_ms": 0.0, "connect_ms": 0.0,
|
||||
"retries_attempted": 0, "retry_succeeded": False, "circuit_open": False}
|
||||
if not self._breaker.allow():
|
||||
timing["circuit_open"] = True
|
||||
log("circuit breaker open, rejecting upstream call")
|
||||
return None, None, timing
|
||||
self.requests += 1
|
||||
self.in_flight += 1
|
||||
if self.in_flight > self.peak_in_flight:
|
||||
self.peak_in_flight = self.in_flight
|
||||
wait_start = time.monotonic()
|
||||
connect_holder = {"ms": 0.0}
|
||||
attempts = 1
|
||||
resp = None
|
||||
exc = None
|
||||
try:
|
||||
async with sem:
|
||||
timing["queue_wait_ms"] = round((time.monotonic() - wait_start) * 1000, 3)
|
||||
|
||||
async def do_call():
|
||||
request = client.build_request(method, url, headers=headers,
|
||||
json=json_body, content=content)
|
||||
request.extensions["trace"] = _connect_tracer(connect_holder)
|
||||
return await client.send(request)
|
||||
|
||||
send_start = time.monotonic()
|
||||
resp, exc, attempts = await retry_send(
|
||||
do_call, cfg["gateway_max_retries"], cfg["gateway_retry_backoff_ms"], log)
|
||||
timing["upstream_latency_ms"] = round((time.monotonic() - send_start) * 1000, 3)
|
||||
finally:
|
||||
self.in_flight -= 1
|
||||
timing["connect_ms"] = round(connect_holder["ms"], 3)
|
||||
timing["retries_attempted"] = max(attempts - 1, 0)
|
||||
self.last_latency_ms = int(timing["upstream_latency_ms"])
|
||||
if exc is not None:
|
||||
self.errors += 1
|
||||
self._breaker.record_failure()
|
||||
log(f"{method} {url} connection failed after {attempts} attempt(s): {exc}")
|
||||
return None, exc, timing
|
||||
self.last_status = resp.status_code
|
||||
if resp.status_code >= 500:
|
||||
self.errors += 1
|
||||
self._breaker.record_failure()
|
||||
else:
|
||||
self._breaker.record_success()
|
||||
timing["retry_succeeded"] = attempts > 1
|
||||
return resp, None, timing
|
||||
|
||||
async def handle_chat(self, body: dict, cfg: dict, owner: tuple, user_agent: str, log=None):
|
||||
log = log or (lambda message: None)
|
||||
client, sem = self._ensure(cfg)
|
||||
pricing = pricing_from_cfg(cfg)
|
||||
context_map = parse_context_map(cfg.get("gateway_model_context_map"))
|
||||
params = extract_params(body)
|
||||
handle_start = time.monotonic()
|
||||
messages = body.get("messages", []) or []
|
||||
|
||||
if cfg["gateway_vision_enabled"]:
|
||||
augmenter = VisionAugmenter(
|
||||
cfg["gateway_vision_url"], cfg["gateway_vision_model"], cfg["gateway_vision_key"],
|
||||
self._vision_cache, ledger=self._ledger, owner=owner, pricing=pricing,
|
||||
context_map=context_map,
|
||||
)
|
||||
messages = await augmenter.augment_messages(client, messages)
|
||||
self.vision_calls += augmenter.calls
|
||||
|
||||
requested = body.get("model")
|
||||
if cfg["gateway_force_model"] or not requested or requested == "molodetz":
|
||||
model = cfg["gateway_model"]
|
||||
else:
|
||||
model = requested
|
||||
|
||||
stream = bool(body.get("stream"))
|
||||
payload = dict(body)
|
||||
payload["model"] = model
|
||||
payload["messages"] = messages
|
||||
payload["stream"] = False
|
||||
|
||||
headers = {"Content-Type": "application/json"}
|
||||
if cfg["gateway_api_key"]:
|
||||
headers["Authorization"] = f"Bearer {cfg['gateway_api_key']}"
|
||||
else:
|
||||
log("No upstream API key configured (gateway_api_key / DEEPSEEK_API_KEY / OPENROUTER_API_KEY); upstream will likely reject the request")
|
||||
|
||||
resp, exc, timing = await self._send(
|
||||
client, sem, "POST", cfg["gateway_upstream_url"], headers, cfg, log, json_body=payload)
|
||||
|
||||
base = {
|
||||
"owner_kind": owner[0], "owner_id": owner[1], "backend": "chat",
|
||||
"endpoint": "chat/completions", "model": model, "user_agent": user_agent,
|
||||
**params, **timing,
|
||||
}
|
||||
|
||||
def finalize(status_code, success, category, usage=None):
|
||||
base["total_latency_ms"] = round((time.monotonic() - handle_start) * 1000, 3)
|
||||
base["gateway_overhead_ms"] = round(max(
|
||||
base["total_latency_ms"] - timing["upstream_latency_ms"] - timing["queue_wait_ms"], 0.0), 3)
|
||||
base["status_code"] = status_code
|
||||
base["success"] = success
|
||||
base["error_category"] = category
|
||||
base["usage"] = usage
|
||||
self._ledger.record(base, pricing, context_map)
|
||||
|
||||
if timing["circuit_open"]:
|
||||
finalize(503, False, "circuit_open")
|
||||
return JSONResponse(status_code=503, content={"error": {"message": "Upstream temporarily unavailable", "type": "circuit_open"}})
|
||||
if exc is not None:
|
||||
finalize(502, False, classify_error(0, exc))
|
||||
return JSONResponse(status_code=502, content={"error": {"message": f"Upstream connection failed: {exc}", "type": "upstream_error"}})
|
||||
if resp.status_code != 200:
|
||||
finalize(resp.status_code, False, classify_error(resp.status_code, None, resp.text))
|
||||
log(f"chat upstream POST -> {resp.status_code}: {resp.text[:300]}")
|
||||
return JSONResponse(status_code=resp.status_code, content={"error": {"message": resp.text, "type": "upstream_error"}})
|
||||
try:
|
||||
data = resp.json()
|
||||
except ValueError:
|
||||
self.errors += 1
|
||||
finalize(502, False, "gateway")
|
||||
log("chat upstream returned 200 but body was not valid JSON")
|
||||
return JSONResponse(status_code=502, content={"error": {"message": "invalid upstream response", "type": "upstream_error"}})
|
||||
finalize(200, True, None, data.get("usage"))
|
||||
log(f"chat POST -> 200 ({timing['upstream_latency_ms']:.0f}ms)")
|
||||
if stream:
|
||||
return StreamingResponse(_fake_stream(data, model), media_type="text/event-stream")
|
||||
return JSONResponse(content=data)
|
||||
|
||||
async def handle_passthrough(self, method: str, subpath: str, content_type: str, body: bytes,
|
||||
cfg: dict, owner: tuple, user_agent: str, log=None):
|
||||
log = log or (lambda message: None)
|
||||
client, sem = self._ensure(cfg)
|
||||
pricing = pricing_from_cfg(cfg)
|
||||
context_map = parse_context_map(cfg.get("gateway_model_context_map"))
|
||||
handle_start = time.monotonic()
|
||||
base_url = cfg["gateway_upstream_url"]
|
||||
if base_url.endswith("/chat/completions"):
|
||||
base_url = base_url[: -len("/chat/completions")]
|
||||
url = f"{base_url.rstrip('/')}/{subpath}"
|
||||
headers = {}
|
||||
if cfg["gateway_api_key"]:
|
||||
headers["Authorization"] = f"Bearer {cfg['gateway_api_key']}"
|
||||
if content_type:
|
||||
headers["Content-Type"] = content_type
|
||||
|
||||
resp, exc, timing = await self._send(client, sem, method, url, headers, cfg, log, content=body)
|
||||
|
||||
base = {
|
||||
"owner_kind": owner[0], "owner_id": owner[1], "backend": "chat",
|
||||
"endpoint": subpath, "model": cfg["gateway_model"], "user_agent": user_agent,
|
||||
**timing,
|
||||
}
|
||||
|
||||
def finalize(status_code, success, category, usage=None):
|
||||
base["total_latency_ms"] = round((time.monotonic() - handle_start) * 1000, 3)
|
||||
base["gateway_overhead_ms"] = round(max(
|
||||
base["total_latency_ms"] - timing["upstream_latency_ms"] - timing["queue_wait_ms"], 0.0), 3)
|
||||
base["status_code"] = status_code
|
||||
base["success"] = success
|
||||
base["error_category"] = category
|
||||
base["usage"] = usage
|
||||
self._ledger.record(base, pricing, context_map)
|
||||
|
||||
if timing["circuit_open"]:
|
||||
finalize(503, False, "circuit_open")
|
||||
return JSONResponse(status_code=503, content={"error": {"message": "Upstream temporarily unavailable", "type": "circuit_open"}})
|
||||
if exc is not None:
|
||||
finalize(502, False, classify_error(0, exc))
|
||||
return JSONResponse(status_code=502, content={"error": {"message": f"Upstream connection failed: {exc}", "type": "upstream_error"}})
|
||||
usage = None
|
||||
if resp.status_code < 400 and "application/json" in (resp.headers.get("content-type") or ""):
|
||||
try:
|
||||
usage = resp.json().get("usage")
|
||||
except ValueError:
|
||||
usage = None
|
||||
finalize(resp.status_code, resp.status_code < 400,
|
||||
None if resp.status_code < 400 else classify_error(resp.status_code, None, resp.text), usage)
|
||||
log(f"passthrough {method} {url} -> {resp.status_code} ({timing['upstream_latency_ms']:.0f}ms)")
|
||||
return Response(content=resp.content, status_code=resp.status_code,
|
||||
media_type=resp.headers.get("content-type"))
|
||||
|
||||
def metrics(self) -> dict:
|
||||
return {
|
||||
"requests": self.requests,
|
||||
"errors": self.errors,
|
||||
"in_flight": self.in_flight,
|
||||
"peak_in_flight": self.peak_in_flight,
|
||||
"vision_calls": self.vision_calls,
|
||||
"last_status": self.last_status,
|
||||
"last_latency_ms": self.last_latency_ms,
|
||||
"pool": self._instances,
|
||||
"circuit_open": self._breaker.is_open,
|
||||
}
|
||||
Reference in New Issue
Block a user