711 lines
26 KiB
Python
711 lines
26 KiB
Python
# retoor <retoor@molodetz.nl>
|
|
|
|
import json
|
|
import logging
|
|
import os
|
|
import re
|
|
import time
|
|
|
|
from fastapi import HTTPException, Request
|
|
from fastapi.responses import JSONResponse
|
|
|
|
from devplacepy.database import get_int_setting
|
|
from devplacepy.services.base import BaseService, ConfigField
|
|
from devplacepy.services.openai_gateway import config, quota
|
|
from devplacepy.services.openai_gateway.analytics import summary_metrics
|
|
from devplacepy.services.openai_gateway.gateway import GatewayRuntime
|
|
from devplacepy.services.openai_gateway.routing import model_store
|
|
from devplacepy.utils import get_current_user
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
APP_REFERENCE_PATTERN = re.compile(r"^[a-zA-Z0-9_.-]{1,30}$")
|
|
DEFAULT_APP_REFERENCE = "default"
|
|
|
|
|
|
def _validate_app_reference(value: str) -> str:
|
|
stripped = (value or "").strip()
|
|
if not stripped or not APP_REFERENCE_PATTERN.match(stripped):
|
|
return DEFAULT_APP_REFERENCE
|
|
return stripped
|
|
|
|
|
|
def _presented_key(request: Request) -> str:
|
|
key = request.headers.get("X-API-KEY")
|
|
if key:
|
|
return key.strip()
|
|
scheme, _, credentials = request.headers.get("Authorization", "").partition(" ")
|
|
if scheme.lower() == "bearer" and credentials.strip():
|
|
return credentials.strip()
|
|
return ""
|
|
|
|
|
|
class GatewayService(BaseService):
|
|
default_enabled = True
|
|
min_interval = 5
|
|
title = "OpenAI Gateway"
|
|
description = (
|
|
"An OpenAI-compatible LLM endpoint at /openai/v1/* that forwards requests to "
|
|
"the configured upstream (DeepSeek by default). Image content is described by "
|
|
"a vision model first so vision-less upstreams still work. Access is gated by "
|
|
"a static key or DevPlace credentials, and throughput scales with the "
|
|
"instances setting."
|
|
)
|
|
config_fields = [
|
|
ConfigField(
|
|
"gateway_upstream_url",
|
|
"Upstream URL",
|
|
type="url",
|
|
default=config.UPSTREAM_URL_DEFAULT,
|
|
help="OpenAI-compatible chat-completions endpoint requests are forwarded to.",
|
|
group="Upstream",
|
|
),
|
|
ConfigField(
|
|
"gateway_model",
|
|
"Model",
|
|
type="str",
|
|
default=config.MODEL_DEFAULT,
|
|
help="Model sent upstream.",
|
|
group="Upstream",
|
|
),
|
|
ConfigField(
|
|
"gateway_force_model",
|
|
"Force model",
|
|
type="bool",
|
|
default=True,
|
|
help="Override the client-requested model with the configured model.",
|
|
group="Upstream",
|
|
),
|
|
ConfigField(
|
|
"gateway_api_key",
|
|
"Upstream API key",
|
|
type="str",
|
|
default="",
|
|
help="The key currently in use; auto-migrated from DEEPSEEK_API_KEY or OPENROUTER_API_KEY on boot. Editable.",
|
|
group="Upstream",
|
|
),
|
|
ConfigField(
|
|
"gateway_timeout",
|
|
"Upstream timeout (seconds)",
|
|
type="int",
|
|
default=config.TIMEOUT_DEFAULT,
|
|
minimum=config.TIMEOUT_MIN,
|
|
help="Per-request upstream timeout. Minimum five minutes.",
|
|
group="Upstream",
|
|
),
|
|
ConfigField(
|
|
"gateway_instances",
|
|
"Instances (concurrency)",
|
|
type="int",
|
|
default=config.INSTANCES_DEFAULT,
|
|
minimum=1,
|
|
maximum=64,
|
|
help="Max concurrent upstream forwards per worker (connection pool + semaphore).",
|
|
group="Upstream",
|
|
),
|
|
ConfigField(
|
|
"gateway_system_preamble",
|
|
"System preamble",
|
|
type="text",
|
|
default=config.SYSTEM_PREAMBLE_DEFAULT,
|
|
help="Operator text prepended ahead of every chat request's system message "
|
|
"(before an auto-injected EU-format date line and the client's own system "
|
|
"content, all in one system message). Leave blank to disable.",
|
|
group="Prompt",
|
|
),
|
|
ConfigField(
|
|
"gateway_vision_enabled",
|
|
"Vision augmentation",
|
|
type="bool",
|
|
default=True,
|
|
help="Describe image content via the vision model before forwarding.",
|
|
group="Vision",
|
|
),
|
|
ConfigField(
|
|
"gateway_vision_url",
|
|
"Vision URL",
|
|
type="url",
|
|
default=config.VISION_URL_DEFAULT,
|
|
help="OpenAI-compatible endpoint used to describe images.",
|
|
group="Vision",
|
|
),
|
|
ConfigField(
|
|
"gateway_vision_model",
|
|
"Vision model",
|
|
type="str",
|
|
default=config.VISION_MODEL_DEFAULT,
|
|
help="Vision-capable model name.",
|
|
group="Vision",
|
|
),
|
|
ConfigField(
|
|
"gateway_vision_key",
|
|
"Vision API key",
|
|
type="str",
|
|
default="",
|
|
help="The key currently in use; auto-migrated from OPENROUTER_API_KEY on boot. Editable.",
|
|
group="Vision",
|
|
),
|
|
ConfigField(
|
|
"gateway_vision_cache_size",
|
|
"Vision cache size",
|
|
type="int",
|
|
default=config.VISION_CACHE_SIZE_DEFAULT,
|
|
minimum=0,
|
|
help="Image-description LRU cache entries (0 disables caching).",
|
|
group="Vision",
|
|
),
|
|
ConfigField(
|
|
"gateway_embed_enabled",
|
|
"Embeddings",
|
|
type="bool",
|
|
default=True,
|
|
help="Expose the embeddings model at /openai/v1/embeddings.",
|
|
group="Embeddings",
|
|
),
|
|
ConfigField(
|
|
"gateway_embed_url",
|
|
"Embeddings URL",
|
|
type="url",
|
|
default=config.EMBED_URL_DEFAULT,
|
|
help="OpenAI-compatible embeddings endpoint requests are forwarded to.",
|
|
group="Embeddings",
|
|
),
|
|
ConfigField(
|
|
"gateway_embed_model",
|
|
"Embeddings model",
|
|
type="str",
|
|
default=config.EMBED_MODEL_DEFAULT,
|
|
help="Embedding model sent upstream. Clients request it as molodetz~embed.",
|
|
group="Embeddings",
|
|
),
|
|
ConfigField(
|
|
"gateway_embed_key",
|
|
"Embeddings API key",
|
|
type="str",
|
|
default="",
|
|
help="The key currently in use; falls back to the vision/OPENROUTER key on boot (embeddings default to OpenRouter). Editable.",
|
|
group="Embeddings",
|
|
),
|
|
ConfigField(
|
|
"gateway_image_enabled",
|
|
"Image generation",
|
|
type="bool",
|
|
default=True,
|
|
help="Expose image generation at /openai/v1/images/generations.",
|
|
group="Images",
|
|
),
|
|
ConfigField(
|
|
"gateway_image_url",
|
|
"Images URL",
|
|
type="url",
|
|
default=config.IMAGE_URL_DEFAULT,
|
|
help="OpenAI-compatible images/generations endpoint requests are forwarded to.",
|
|
group="Images",
|
|
),
|
|
ConfigField(
|
|
"gateway_image_model",
|
|
"Images model",
|
|
type="str",
|
|
default=config.IMAGE_MODEL_DEFAULT,
|
|
help="Image model sent upstream. Clients request it as molodetz-img-small.",
|
|
group="Images",
|
|
),
|
|
ConfigField(
|
|
"gateway_image_key",
|
|
"Images API key",
|
|
type="str",
|
|
default="",
|
|
help="The key currently in use; falls back to gateway_api_key / OPENROUTER_API_KEY on boot. Editable.",
|
|
group="Images",
|
|
),
|
|
ConfigField(
|
|
"gateway_require_auth",
|
|
"Require authentication",
|
|
type="bool",
|
|
default=True,
|
|
help="When off, the gateway is open to anyone.",
|
|
group="Access",
|
|
),
|
|
ConfigField(
|
|
"gateway_allow_admins",
|
|
"Allow admins",
|
|
type="bool",
|
|
default=True,
|
|
help="Admin users (API key / Bearer / Basic / session) may call the gateway.",
|
|
group="Access",
|
|
),
|
|
ConfigField(
|
|
"gateway_allow_users",
|
|
"Allow users",
|
|
type="bool",
|
|
default=True,
|
|
help="Any authenticated user may call the gateway with their own API key. "
|
|
"Devii operates a signed-in user's account with that user's key, so usage is "
|
|
"attributed and limitable per user.",
|
|
group="Access",
|
|
),
|
|
ConfigField(
|
|
"gateway_access_key",
|
|
"Static access key",
|
|
type="password",
|
|
default="",
|
|
secret=True,
|
|
help="A standalone key that always grants access (sent as X-API-KEY or Bearer).",
|
|
group="Access",
|
|
),
|
|
ConfigField(
|
|
"gateway_internal_key",
|
|
"Internal key",
|
|
type="str",
|
|
default="",
|
|
help="Auto-generated on boot. DevPlace's own services authenticate to the gateway "
|
|
"with this key. Clear it and restart to rotate.",
|
|
group="Access",
|
|
),
|
|
ConfigField(
|
|
quota.FIELD_DEFAULT_USER,
|
|
"Default per-user daily cap ($)",
|
|
type="float",
|
|
default=1.0,
|
|
minimum=0,
|
|
help="Rolling 24h cap applied to a signed-in member with no matching quota rule. "
|
|
"0 = unlimited. Overridable per user/app-reference on /admin/gateway.",
|
|
group="Quota",
|
|
),
|
|
ConfigField(
|
|
quota.FIELD_DEFAULT_ADMIN,
|
|
"Default per-admin daily cap ($)",
|
|
type="float",
|
|
default=0.0,
|
|
minimum=0,
|
|
help="Rolling 24h cap applied to an administrator with no matching quota rule. "
|
|
"0 = unlimited (the default - admins are exempt unless a rule says otherwise).",
|
|
group="Quota",
|
|
),
|
|
ConfigField(
|
|
quota.FIELD_DEFAULT_GUEST,
|
|
"Default per-guest daily cap ($)",
|
|
type="float",
|
|
default=0.05,
|
|
minimum=0,
|
|
help="Rolling 24h cap applied to an unauthenticated caller with no matching quota "
|
|
"rule (only reachable when authentication is not required). 0 = unlimited.",
|
|
group="Quota",
|
|
),
|
|
ConfigField(
|
|
quota.FIELD_DEFAULT_INTERNAL,
|
|
"Default per-internal-key daily cap ($)",
|
|
type="float",
|
|
default=0.0,
|
|
minimum=0,
|
|
help="Rolling 24h cap applied to calls authenticated with the auto-generated internal "
|
|
"key (DevPlace's own services: news, bots, Devii guests, correction/modifier). "
|
|
"0 = unlimited (the default - do not cap this without a matching quota rule, or "
|
|
"internal platform traffic will start failing).",
|
|
group="Quota",
|
|
),
|
|
ConfigField(
|
|
quota.FIELD_DEFAULT_KEY,
|
|
"Default per-access-key daily cap ($)",
|
|
type="float",
|
|
default=0.0,
|
|
minimum=0,
|
|
help="Rolling 24h cap applied to calls authenticated with the static access key. "
|
|
"0 = unlimited by default (it is an admin-provisioned trusted secret).",
|
|
group="Quota",
|
|
),
|
|
ConfigField(
|
|
"gateway_price_cache_hit_per_m",
|
|
"Chat price cache-hit / 1M ($)",
|
|
type="float",
|
|
default=config.PRICE_CACHE_HIT_PER_M_DEFAULT,
|
|
minimum=0,
|
|
help="Estimates chat cost when the upstream returns no native cost field (DeepSeek).",
|
|
group="Pricing",
|
|
),
|
|
ConfigField(
|
|
"gateway_price_cache_miss_per_m",
|
|
"Chat price cache-miss / 1M ($)",
|
|
type="float",
|
|
default=config.PRICE_CACHE_MISS_PER_M_DEFAULT,
|
|
minimum=0,
|
|
group="Pricing",
|
|
),
|
|
ConfigField(
|
|
"gateway_price_output_per_m",
|
|
"Chat price output / 1M ($)",
|
|
type="float",
|
|
default=config.PRICE_OUTPUT_PER_M_DEFAULT,
|
|
minimum=0,
|
|
group="Pricing",
|
|
),
|
|
ConfigField(
|
|
"gateway_vision_price_input_per_m",
|
|
"Vision price input / 1M ($)",
|
|
type="float",
|
|
default=config.VISION_PRICE_INPUT_PER_M_DEFAULT,
|
|
minimum=0,
|
|
help="Fallback only; used when the vision upstream returns no native cost.",
|
|
group="Pricing",
|
|
),
|
|
ConfigField(
|
|
"gateway_vision_price_output_per_m",
|
|
"Vision price output / 1M ($)",
|
|
type="float",
|
|
default=config.VISION_PRICE_OUTPUT_PER_M_DEFAULT,
|
|
minimum=0,
|
|
group="Pricing",
|
|
),
|
|
ConfigField(
|
|
"gateway_embed_price_input_per_m",
|
|
"Embeddings price input / 1M ($)",
|
|
type="float",
|
|
default=config.EMBED_PRICE_INPUT_PER_M_DEFAULT,
|
|
minimum=0,
|
|
help="Fallback only; used when the embeddings upstream returns no native cost.",
|
|
group="Pricing",
|
|
),
|
|
ConfigField(
|
|
"gateway_image_price_per_call",
|
|
"Images price / call ($)",
|
|
type="float",
|
|
default=config.IMAGE_PRICE_PER_CALL_DEFAULT,
|
|
minimum=0,
|
|
help="Fallback only; used when the image upstream returns no native cost.",
|
|
group="Pricing",
|
|
),
|
|
ConfigField(
|
|
"gateway_rsearch_cost_per_call",
|
|
"rsearch cost / call ($)",
|
|
type="float",
|
|
default=config.RSEARCH_COST_PER_CALL_DEFAULT,
|
|
minimum=0,
|
|
help="Flat cost attributed to each external rsearch call (web search / AI answer / chat / image describe), recorded under backend 'rsearch' so external AI spend appears in AI usage.",
|
|
group="Pricing",
|
|
),
|
|
ConfigField(
|
|
"gateway_max_retries",
|
|
"Max retries",
|
|
type="int",
|
|
default=config.MAX_RETRIES_DEFAULT,
|
|
minimum=0,
|
|
maximum=10,
|
|
help="Retry attempts on timeout, connection error, or upstream 5xx.",
|
|
group="Reliability",
|
|
),
|
|
ConfigField(
|
|
"gateway_retry_backoff_ms",
|
|
"Retry backoff (ms)",
|
|
type="int",
|
|
default=config.RETRY_BACKOFF_MS_DEFAULT,
|
|
minimum=0,
|
|
help="Linear backoff multiplied by the attempt number.",
|
|
group="Reliability",
|
|
),
|
|
ConfigField(
|
|
"gateway_circuit_threshold",
|
|
"Circuit breaker threshold",
|
|
type="int",
|
|
default=config.CIRCUIT_THRESHOLD_DEFAULT,
|
|
minimum=0,
|
|
help="Consecutive upstream failures before the breaker opens (0 disables).",
|
|
group="Reliability",
|
|
),
|
|
ConfigField(
|
|
"gateway_circuit_cooldown_seconds",
|
|
"Circuit breaker cooldown (s)",
|
|
type="int",
|
|
default=config.CIRCUIT_COOLDOWN_SECONDS_DEFAULT,
|
|
minimum=1,
|
|
group="Reliability",
|
|
),
|
|
ConfigField(
|
|
"gateway_usage_retention_hours",
|
|
"Usage retention (hours)",
|
|
type="int",
|
|
default=config.USAGE_RETENTION_HOURS_DEFAULT,
|
|
minimum=1,
|
|
help="How long per-call usage rows are kept before pruning.",
|
|
group="Tracking",
|
|
),
|
|
ConfigField(
|
|
"gateway_model_context_map",
|
|
"Model context map (JSON)",
|
|
type="str",
|
|
default=json.dumps(config.MODEL_CONTEXT_MAP_DEFAULT),
|
|
help="JSON object mapping model name to max context tokens for utilization tracking.",
|
|
group="Tracking",
|
|
),
|
|
]
|
|
|
|
def __init__(self):
|
|
super().__init__(name="openai", interval_seconds=30)
|
|
self._runtime = None
|
|
|
|
def runtime(self) -> GatewayRuntime:
|
|
if self._runtime is None:
|
|
self._runtime = GatewayRuntime()
|
|
return self._runtime
|
|
|
|
def effective_config(self) -> dict:
|
|
cfg = self.get_config()
|
|
cfg["gateway_api_key"] = (
|
|
cfg["gateway_api_key"]
|
|
or os.environ.get("DEEPSEEK_API_KEY", "")
|
|
or os.environ.get("OPENROUTER_API_KEY", "")
|
|
)
|
|
cfg["gateway_vision_key"] = cfg["gateway_vision_key"] or os.environ.get(
|
|
"OPENROUTER_API_KEY", ""
|
|
)
|
|
cfg["gateway_embed_key"] = (
|
|
cfg["gateway_embed_key"]
|
|
or cfg["gateway_vision_key"]
|
|
or os.environ.get("OPENROUTER_API_KEY", "")
|
|
)
|
|
cfg["gateway_image_key"] = (
|
|
cfg["gateway_image_key"]
|
|
or cfg["gateway_api_key"]
|
|
or os.environ.get("OPENROUTER_API_KEY", "")
|
|
)
|
|
if not cfg.get("gateway_image_url"):
|
|
upstream = cfg.get("gateway_upstream_url", "")
|
|
if upstream.endswith("/chat/completions"):
|
|
cfg["gateway_image_url"] = (
|
|
upstream[: -len("/chat/completions")] + "/images"
|
|
)
|
|
return cfg
|
|
|
|
def authorize(self, request: Request) -> bool:
|
|
cfg = self.get_config()
|
|
if not cfg["gateway_require_auth"]:
|
|
return True
|
|
access_key = cfg["gateway_access_key"]
|
|
internal_key = cfg["gateway_internal_key"]
|
|
presented = _presented_key(request)
|
|
if presented and access_key and presented == access_key:
|
|
return True
|
|
if presented and internal_key and presented == internal_key:
|
|
return True
|
|
user = get_current_user(request)
|
|
if user:
|
|
if user.get("role") == "Admin" and cfg["gateway_allow_admins"]:
|
|
return True
|
|
if cfg["gateway_allow_users"]:
|
|
return True
|
|
return False
|
|
|
|
def resolve_owner(self, request: Request) -> tuple:
|
|
cfg = self.get_config()
|
|
presented = _presented_key(request)
|
|
if (
|
|
presented
|
|
and cfg["gateway_internal_key"]
|
|
and presented == cfg["gateway_internal_key"]
|
|
):
|
|
return ("internal", "devii")
|
|
if (
|
|
presented
|
|
and cfg["gateway_access_key"]
|
|
and presented == cfg["gateway_access_key"]
|
|
):
|
|
return ("key", "access")
|
|
user = get_current_user(request)
|
|
if user:
|
|
kind = "admin" if user.get("role") == "Admin" else "user"
|
|
return (kind, user.get("uid") or "unknown")
|
|
return ("anonymous", "anonymous")
|
|
|
|
def _audit_quota_exceeded(
|
|
self,
|
|
owner_kind: str,
|
|
owner_id: str,
|
|
app_reference: str,
|
|
spent: float,
|
|
limit: float,
|
|
rule,
|
|
) -> None:
|
|
from devplacepy.services.audit import record as audit
|
|
from devplacepy.services.openai_gateway.usage import audit_actor_for
|
|
|
|
actor_kind, actor_uid, actor_role = audit_actor_for(owner_kind, owner_id)
|
|
audit.record_system(
|
|
"ai.quota.exceeded",
|
|
actor_kind=actor_kind,
|
|
actor_uid=actor_uid,
|
|
actor_role=actor_role,
|
|
origin="api",
|
|
result="denied",
|
|
summary=f"AI gateway call by {owner_kind}/{owner_id} blocked - 24h quota reached",
|
|
metadata={
|
|
"owner_kind": owner_kind,
|
|
"owner_id": owner_id,
|
|
"app_reference": app_reference,
|
|
"spent_usd": round(spent, 6),
|
|
"limit_usd": limit,
|
|
"rule_uid": rule.uid if rule else None,
|
|
},
|
|
)
|
|
|
|
def _models_response(self) -> JSONResponse:
|
|
created = int(time.time())
|
|
seen: set = set()
|
|
data = []
|
|
for row in model_store.list():
|
|
if str(row.get("kind") or "chat") != "chat":
|
|
continue
|
|
if not row.get("is_active", True):
|
|
continue
|
|
source = str(row.get("source_model") or "").strip()
|
|
if not source or source in seen:
|
|
continue
|
|
seen.add(source)
|
|
data.append(
|
|
{
|
|
"id": source,
|
|
"object": "model",
|
|
"created": created,
|
|
"owned_by": "molodetz",
|
|
}
|
|
)
|
|
return JSONResponse({"object": "list", "data": data})
|
|
|
|
async def handle(self, request: Request, subpath: str):
|
|
if not self.is_enabled():
|
|
raise HTTPException(status_code=503, detail="Gateway is disabled")
|
|
if not self.authorize(request):
|
|
self.log(f"Rejected {request.method} /{subpath}: unauthorized")
|
|
raise HTTPException(status_code=401, detail="Unauthorized")
|
|
cfg = self.effective_config()
|
|
runtime = self.runtime()
|
|
owner = self.resolve_owner(request)
|
|
user_agent = request.headers.get("user-agent", "")
|
|
app_reference = _validate_app_reference(
|
|
request.headers.get("X-App-Reference", DEFAULT_APP_REFERENCE)
|
|
)
|
|
if subpath == "models" and request.method == "GET":
|
|
return self._models_response()
|
|
limit, scope, rule = quota.resolve(owner[0], owner[1], app_reference, cfg)
|
|
if limit > 0:
|
|
spent = quota.spent_24h(*scope)
|
|
if spent >= limit:
|
|
self.log(
|
|
f"Rejected {owner[0]}:{owner[1]} app={app_reference}: "
|
|
f"quota exceeded (${spent:.4f}/${limit:.4f}"
|
|
f"{' rule ' + rule.uid if rule else ' default'})"
|
|
)
|
|
self._audit_quota_exceeded(owner[0], owner[1], app_reference, spent, limit, rule)
|
|
raise HTTPException(status_code=429, detail="AI gateway daily quota exceeded")
|
|
if subpath == "chat/completions" and request.method == "POST":
|
|
try:
|
|
body = await request.json()
|
|
except Exception:
|
|
self.log("Rejected chat request: invalid JSON body")
|
|
raise HTTPException(status_code=400, detail="Invalid JSON body")
|
|
if not isinstance(body, dict):
|
|
self.log("Rejected chat request: JSON body was not an object")
|
|
raise HTTPException(status_code=400, detail="Invalid JSON body")
|
|
return await runtime.handle_chat(body, cfg, owner, user_agent, app_reference, self.log)
|
|
if subpath == "embeddings" and request.method == "POST":
|
|
try:
|
|
body = await request.json()
|
|
except Exception:
|
|
self.log("Rejected embeddings request: invalid JSON body")
|
|
raise HTTPException(status_code=400, detail="Invalid JSON body")
|
|
if not isinstance(body, dict):
|
|
self.log("Rejected embeddings request: JSON body was not an object")
|
|
raise HTTPException(status_code=400, detail="Invalid JSON body")
|
|
return await runtime.handle_embeddings(
|
|
body, cfg, owner, user_agent, app_reference, self.log
|
|
)
|
|
if subpath == "images/generations" and request.method == "POST":
|
|
try:
|
|
body = await request.json()
|
|
except Exception:
|
|
self.log("Rejected images request: invalid JSON body")
|
|
raise HTTPException(status_code=400, detail="Invalid JSON body")
|
|
if not isinstance(body, dict):
|
|
self.log("Rejected images request: JSON body was not an object")
|
|
raise HTTPException(status_code=400, detail="Invalid JSON body")
|
|
return await runtime.handle_images(
|
|
body, cfg, owner, user_agent, app_reference, self.log
|
|
)
|
|
body = await request.body()
|
|
content_type = request.headers.get("content-type", "")
|
|
return await runtime.handle_passthrough(
|
|
request.method,
|
|
subpath,
|
|
content_type,
|
|
body,
|
|
cfg,
|
|
owner,
|
|
user_agent,
|
|
app_reference,
|
|
self.log,
|
|
)
|
|
|
|
async def run_once(self) -> None:
|
|
if not self.is_enabled():
|
|
return
|
|
runtime = self.runtime()
|
|
runtime._ensure(self.effective_config())
|
|
runtime._ledger.sample_concurrency(runtime.in_flight)
|
|
retention = get_int_setting(
|
|
"gateway_usage_retention_hours", config.USAGE_RETENTION_HOURS_DEFAULT
|
|
)
|
|
ledger_removed, samples_removed = runtime._ledger.prune(retention)
|
|
if ledger_removed or samples_removed:
|
|
self.log(
|
|
f"Pruned {ledger_removed} usage rows and {samples_removed} concurrency samples"
|
|
)
|
|
|
|
async def on_disable(self) -> None:
|
|
if self._runtime is not None:
|
|
await self._runtime.aclose()
|
|
|
|
def collect_metrics(self) -> dict:
|
|
cfg = self.get_config()
|
|
m = (
|
|
self._runtime.metrics()
|
|
if self._runtime is not None
|
|
else {
|
|
"requests": 0,
|
|
"errors": 0,
|
|
"in_flight": 0,
|
|
"peak_in_flight": 0,
|
|
"vision_calls": 0,
|
|
"embed_calls": 0,
|
|
"image_calls": 0,
|
|
"last_status": 0,
|
|
"last_latency_ms": 0,
|
|
"pool": 0,
|
|
"circuit_open": False,
|
|
}
|
|
)
|
|
s = summary_metrics()
|
|
stats = [
|
|
{"label": "Requests (lifetime)", "value": m["requests"]},
|
|
{"label": "In flight", "value": m["in_flight"]},
|
|
{"label": "Vision calls", "value": m["vision_calls"]},
|
|
{"label": "Embed calls", "value": m["embed_calls"]},
|
|
{"label": "Image calls", "value": m["image_calls"]},
|
|
{"label": "Last status", "value": m["last_status"] or "-"},
|
|
{"label": "Last latency", "value": f"{m['last_latency_ms']} ms"},
|
|
{"label": "Pool size", "value": m["pool"]},
|
|
{"label": "Circuit", "value": "open" if m["circuit_open"] else "closed"},
|
|
{"label": "Model", "value": cfg["gateway_model"]},
|
|
{"label": "Embed model", "value": cfg["gateway_embed_model"]},
|
|
{"label": "Image model", "value": cfg["gateway_image_model"]},
|
|
{"label": "Requests 24h", "value": s["requests"]},
|
|
{"label": "Success 24h", "value": f"{s['success_pct']}%"},
|
|
{"label": "Error rate 24h", "value": f"{s['error_pct']}%"},
|
|
{"label": "Cost this hour", "value": f"${s['cost_hour']:.4f}"},
|
|
{"label": "Cost 24h", "value": f"${s['cost_24h']:.2f}"},
|
|
{"label": "Tokens 24h", "value": s["tokens_24h"]},
|
|
{"label": "Avg latency 24h", "value": f"{s['avg_latency_ms']:.0f} ms"},
|
|
{"label": "Avg tokens/s 24h", "value": s["avg_tps"]},
|
|
{"label": "Peak concurrency 24h", "value": s["peak_concurrency"]},
|
|
{"label": "Top model 24h", "value": s["top_model"]},
|
|
{"label": "Top caller 24h", "value": s["top_caller"]},
|
|
]
|
|
return {"stats": stats}
|