158 lines
5.1 KiB
Python
158 lines
5.1 KiB
Python
# retoor <retoor@molodetz.nl>
|
|
|
|
from __future__ import annotations
|
|
|
|
import asyncio
|
|
import logging
|
|
import time
|
|
from datetime import datetime, timezone
|
|
from typing import Awaitable, Callable, Optional
|
|
|
|
import httpx
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
|
|
def percentile(sorted_values: list[float], q: float) -> float:
|
|
if not sorted_values:
|
|
return 0.0
|
|
if len(sorted_values) == 1:
|
|
return float(sorted_values[0])
|
|
rank = (len(sorted_values) - 1) * q
|
|
low = int(rank)
|
|
high = min(low + 1, len(sorted_values) - 1)
|
|
frac = rank - low
|
|
return float(sorted_values[low] + (sorted_values[high] - sorted_values[low]) * frac)
|
|
|
|
|
|
class CircuitBreaker:
|
|
def __init__(self, threshold: int, cooldown_seconds: int):
|
|
self.threshold = threshold
|
|
self.cooldown_seconds = cooldown_seconds
|
|
self.failures = 0
|
|
self.opened_at: Optional[float] = None
|
|
|
|
def configure(self, threshold: int, cooldown_seconds: int) -> None:
|
|
self.threshold = threshold
|
|
self.cooldown_seconds = cooldown_seconds
|
|
|
|
@property
|
|
def is_open(self) -> bool:
|
|
return self.opened_at is not None
|
|
|
|
def allow(self) -> bool:
|
|
if self.opened_at is None:
|
|
return True
|
|
if (time.monotonic() - self.opened_at) >= self.cooldown_seconds:
|
|
self.opened_at = None
|
|
self.failures = 0
|
|
return True
|
|
return False
|
|
|
|
def record_success(self) -> None:
|
|
self.failures = 0
|
|
self.opened_at = None
|
|
|
|
def record_failure(self) -> None:
|
|
self.failures += 1
|
|
if self.threshold > 0 and self.failures >= self.threshold:
|
|
self.opened_at = time.monotonic()
|
|
|
|
def trip(self) -> None:
|
|
self.failures = max(self.threshold, 1)
|
|
self.opened_at = time.monotonic()
|
|
|
|
|
|
MAX_RETRY_AFTER_SECONDS = 30.0
|
|
|
|
|
|
def _retry_after_seconds(resp: httpx.Response) -> Optional[float]:
|
|
# DeepSeek documents no Retry-After; OpenRouter documents a standard
|
|
# HTTP Retry-After (seconds, sometimes an HTTP-date) on 429/503 and
|
|
# instructs clients to honor it - see openrouter.ai/docs/api-reference/errors.
|
|
raw = resp.headers.get("retry-after")
|
|
if not raw:
|
|
return None
|
|
try:
|
|
seconds = float(raw)
|
|
except ValueError:
|
|
try:
|
|
from email.utils import parsedate_to_datetime
|
|
|
|
when = parsedate_to_datetime(raw)
|
|
if when.tzinfo is None:
|
|
return None
|
|
seconds = (when - datetime.now(timezone.utc)).total_seconds()
|
|
except (TypeError, ValueError, IndexError):
|
|
return None
|
|
if seconds < 0:
|
|
return 0.0
|
|
return min(seconds, MAX_RETRY_AFTER_SECONDS)
|
|
|
|
|
|
async def _backoff(
|
|
backoff_ms: int, attempt: int, retry_after: Optional[float] = None
|
|
) -> None:
|
|
if retry_after is not None:
|
|
if retry_after > 0:
|
|
await asyncio.sleep(retry_after)
|
|
return
|
|
delay = max(0, backoff_ms) * attempt / 1000.0
|
|
if delay > 0:
|
|
await asyncio.sleep(delay)
|
|
|
|
|
|
def _is_retryable_status(status_code: int) -> bool:
|
|
# 5xx is a transient upstream fault. 429 is a rate limit - both DeepSeek
|
|
# and OpenRouter use it and OpenRouter explicitly documents retrying it
|
|
# (with Retry-After honored); a single retry after backoff is safe since
|
|
# gateway_max_retries already bounds the total attempts.
|
|
return status_code >= 500 or status_code == 429
|
|
|
|
|
|
async def retry_send(
|
|
do_call: Callable[[], Awaitable[httpx.Response]],
|
|
sem: asyncio.Semaphore,
|
|
max_retries: int,
|
|
backoff_ms: int,
|
|
log: Optional[Callable[[str], None]] = None,
|
|
) -> tuple[Optional[httpx.Response], Optional[Exception], int, float]:
|
|
log = log or (lambda message: None)
|
|
attempts = 0
|
|
last_exc: Optional[Exception] = None
|
|
queue_wait_ms = 0.0
|
|
while attempts <= max_retries:
|
|
attempts += 1
|
|
wait_start = time.monotonic()
|
|
async with sem:
|
|
queue_wait_ms += (time.monotonic() - wait_start) * 1000
|
|
try:
|
|
resp = await do_call()
|
|
exc = None
|
|
except httpx.RequestError as e:
|
|
resp = None
|
|
exc = e
|
|
if exc is not None:
|
|
last_exc = exc
|
|
if attempts > max_retries:
|
|
return None, exc, attempts, queue_wait_ms
|
|
log(
|
|
f"upstream connection failed, retrying ({attempts}/{max_retries}): {exc}"
|
|
)
|
|
await _backoff(backoff_ms, attempts)
|
|
continue
|
|
if _is_retryable_status(resp.status_code) and attempts <= max_retries:
|
|
retry_after = _retry_after_seconds(resp)
|
|
log(
|
|
f"upstream {resp.status_code}, retrying ({attempts}/{max_retries})"
|
|
+ (f" after Retry-After={retry_after:.1f}s" if retry_after else "")
|
|
)
|
|
try:
|
|
await resp.aclose()
|
|
except Exception: # noqa: BLE001 - releasing the connection must never block a retry
|
|
pass
|
|
await _backoff(backoff_ms, attempts, retry_after)
|
|
continue
|
|
return resp, None, attempts, queue_wait_ms
|
|
return None, last_exc, attempts, queue_wait_ms
|