Files
devplacepy/devplacepy/services/openai_gateway/vision.py
T

372 lines
14 KiB
Python

# retoor <retoor@molodetz.nl>
import asyncio
import hashlib
import logging
import time
from collections import OrderedDict
from typing import Any, Optional
from urllib.parse import urlparse
import httpx
from devplacepy.services.openai_gateway.apps import LEGACY_TITLE_HEADER, TITLE_HEADER
from devplacepy.services.openai_gateway.config import VISION_INSTRUCTION
from devplacepy.services.openai_gateway.reliability import (
_backoff,
_is_retryable_status,
_retry_after_seconds,
)
from devplacepy.services.openai_gateway.learned import learned_store
from devplacepy.services.openai_gateway.shape import ShapeRetrier
from devplacepy.services.openai_gateway.thinking import upstream_capabilities
from devplacepy.services.openai_gateway.usage import classify_error
logger = logging.getLogger(__name__)
class VisionCache:
def __init__(self, size: int = 256):
self.size = size
self._store: "OrderedDict[str, str]" = OrderedDict()
def get(self, key: str) -> Optional[str]:
if self.size <= 0:
return None
value = self._store.get(key)
if value is not None:
self._store.move_to_end(key)
return value
def put(self, key: str, value: str) -> None:
if self.size <= 0:
return
self._store[key] = value
self._store.move_to_end(key)
while len(self._store) > self.size:
self._store.popitem(last=False)
def __len__(self) -> int:
return len(self._store)
def has_vision_blocks(content: Any) -> bool:
if not isinstance(content, list):
return False
return any(
isinstance(b, dict) and b.get("type") in ("image_url", "image") for b in content
)
def split_text_and_images(content: list) -> tuple[str, list]:
texts: list[str] = []
images: list[dict] = []
for block in content:
if not isinstance(block, dict):
continue
btype = block.get("type")
if btype == "text":
t = block.get("text", "")
if t:
texts.append(t)
elif btype in ("image_url", "image"):
images.append(block)
return "\n".join(texts).strip(), images
def _image_cache_key(image_block: dict) -> str:
iu = image_block.get("image_url") or image_block.get("image") or ""
url = iu.get("url", "") if isinstance(iu, dict) else str(iu)
return hashlib.sha256(url.encode("utf-8", errors="replace")).hexdigest()[:32]
def _format_vision_block(descriptions: list) -> str:
if len(descriptions) == 1:
return f"[Image seen by vision model:\n{descriptions[0]}\n]"
parts = [f"Image {i}:\n{d}" for i, d in enumerate(descriptions, 1)]
return (
f"[{len(descriptions)} images seen by vision model:\n"
+ "\n\n".join(parts)
+ "\n]"
)
class VisionAugmenter:
def __init__(
self,
vision_url: str,
vision_model: str,
vision_key: str,
cache: VisionCache,
referer: str = "",
title: str = "",
vision_dialect: str = "",
ledger=None,
owner: tuple = ("unknown", "unknown"),
pricing=None,
context_map=None,
app_reference: str = "default",
breaker=None,
max_retries: int = 2,
retry_backoff_ms: int = 250,
trip_sink=None,
):
self.vision_url = vision_url
self.vision_model = vision_model
self.vision_key = vision_key
self.cache = cache
self.referer = referer
self.title = title
self.vision_dialect = vision_dialect
self.ledger = ledger
self.owner = owner
self.pricing = pricing
self.context_map = context_map or {}
self.app_reference = app_reference
self.breaker = breaker
self.max_retries = max(0, int(max_retries or 0))
self.retry_backoff_ms = max(0, int(retry_backoff_ms or 0))
self.trip_sink = trip_sink
self.calls = 0
self.cost_usd = 0.0
def _record(self, latency_ms, status_code, success, category, usage):
if self.ledger is None or self.pricing is None:
return
row = self.ledger.record(
{
"owner_kind": self.owner[0],
"owner_id": self.owner[1],
"backend": "vision",
"endpoint": "chat/completions",
"model": self.vision_model,
"requested_model": self.vision_model,
"temperature": 0.2,
"upstream_latency_ms": latency_ms,
"total_latency_ms": latency_ms,
"status_code": status_code,
"success": success,
"error_category": category,
"usage": usage,
"app_reference": self.app_reference,
},
self.pricing,
self.context_map,
)
if row:
self.cost_usd += float(row.get("cost_usd") or 0.0)
async def _post_with_retries(
self, client: httpx.AsyncClient, payload: dict, headers: dict
) -> tuple:
attempt = 0
resp: Optional[httpx.Response] = None
exc: Optional[Exception] = None
while True:
attempt += 1
try:
resp = await client.post(self.vision_url, json=payload, headers=headers)
exc = None
except httpx.RequestError as e:
resp = None
exc = e
if exc is None and not _is_retryable_status(resp.status_code):
break
if attempt > self.max_retries:
break
retry_after = _retry_after_seconds(resp) if resp is not None else None
logger.info(
"vision call retrying (%d/%d): %s",
attempt,
self.max_retries,
exc or f"HTTP {resp.status_code}",
)
if resp is not None:
try:
await resp.aclose()
except Exception: # noqa: BLE001
pass
await _backoff(self.retry_backoff_ms, attempt, retry_after)
return resp, exc, attempt
def _report_trip(self, exc, resp, tries) -> None:
try:
if self.breaker is None:
return
if exc is not None:
category = classify_error(0, exc)
status: Any = "no response"
upstream = str(exc)
else:
try:
text = resp.text if resp is not None else ""
except Exception:
text = ""
code = resp.status_code if resp is not None else 0
category = classify_error(code, None, text)
status = code
upstream = text
detail = {
"provider": urlparse(self.vision_url or "").netloc,
"model": self.vision_model,
"route": f"vision:{self.vision_model}",
"cooldown": self.breaker.cooldown_seconds,
"category": category,
"status": status,
"upstream": (upstream or "")[:300],
"retries": max(0, int(tries or 0) - 1),
}
logger.warning(
"ai.gateway.circuit.vision_opened provider=%s model=%s route=%s category=%s status=%s retries=%s",
detail["provider"], detail["model"], detail["route"], category, status, detail["retries"],
)
if self.trip_sink is not None:
self.trip_sink.append({"kind": "vision", "detail": detail})
else:
from devplacepy.services.background import background
from devplacepy.services.openai_gateway import gateway as gateway_mod
background.submit(gateway_mod._notify_gateway_status, True, detail)
except Exception as report_exc:
logger.warning("ai.gateway.circuit.vision report failed: %s", report_exc)
async def _describe_one(self, client: httpx.AsyncClient, image_block: dict) -> str:
if not self.vision_key:
return "[vision unavailable: vision API key not configured]"
if self.breaker is not None and not self.breaker.allow():
self._record((0.0), 503, False, "circuit_open", None)
return "[vision unavailable: circuit breaker open]"
payload = {
"model": self.vision_model,
"messages": [
{
"role": "user",
"content": [
{"type": "text", "text": VISION_INSTRUCTION},
image_block,
],
}
],
"temperature": 0.2,
"stream": False,
}
shape_cfg = {
"gateway_upstream_url": self.vision_url,
"gateway_thinking_dialect": self.vision_dialect or "auto",
"gateway_thinking": False,
}
retrier = ShapeRetrier.for_route(
learned_store,
"vision:" + (urlparse(self.vision_url).hostname or "default"),
self.vision_model,
"chat_completions",
upstream_capabilities(self.vision_url, self.vision_dialect).dialect,
logger.info,
)
headers = {
"Authorization": f"Bearer {self.vision_key}",
"Content-Type": "application/json",
}
if self.referer:
headers["HTTP-Referer"] = self.referer
if self.title:
headers[TITLE_HEADER] = self.title
headers[LEGACY_TITLE_HEADER] = self.title
start = time.monotonic()
shaped = retrier.method.apply(payload, shape_cfg, "chat_completions", False, logger.info)
resp, exc, tries = await self._post_with_retries(client, shaped, headers)
first = (resp, exc)
resolving = False
total_tries = tries
while resp is not None and retrier.eligible(resp.status_code):
method = retrier.next_after(resp.status_code, resp.text)
if method is None:
break
resolving = True
shaped = method.apply(payload, shape_cfg, "chat_completions", False, logger.info)
resp, exc, tries = await self._post_with_retries(client, shaped, headers)
total_tries += tries
if resolving:
if resp is not None and resp.status_code < 400:
retrier.succeeded()
else:
retrier.exhausted()
resp, exc = first
if self.breaker is not None:
if exc is not None or (resp is not None and resp.status_code >= 500):
was_open = self.breaker.is_open
self.breaker.record_failure()
if not was_open and self.breaker.is_open:
self._report_trip(exc, resp, total_tries)
else:
self.breaker.record_success()
if exc is not None:
logger.warning("vision connection failed: %s", exc)
self._record(
(time.monotonic() - start) * 1000,
502,
False,
classify_error(0, exc),
None,
)
return f"[vision call failed: {exc}]"
latency_ms = (time.monotonic() - start) * 1000
if resp.status_code != 200:
logger.warning("vision %s: %s", resp.status_code, resp.text[:200])
self._record(
latency_ms,
resp.status_code,
False,
classify_error(resp.status_code, None, resp.text),
None,
)
return f"[vision failed: HTTP {resp.status_code}]"
try:
data = resp.json()
error = data.get("error") if isinstance(data, dict) else None
if error and "choices" not in data:
message = error.get("message") if isinstance(error, dict) else str(error)
self._record(latency_ms, 502, False, "upstream_error", None)
return f"[vision failed: {message}]"
text = data["choices"][0]["message"].get("content") or ""
self._record(latency_ms, 200, True, None, data.get("usage"))
return text.strip() or "[vision returned empty response]"
except (KeyError, IndexError, ValueError, httpx.HTTPError) as e:
self._record(latency_ms, 200, False, "gateway", None)
return f"[vision parse error: {e}]"
async def _describe_images(self, client: httpx.AsyncClient, images: list) -> list:
keys = [_image_cache_key(img) for img in images]
descriptions: list = [self.cache.get(k) for k in keys]
miss = [i for i, d in enumerate(descriptions) if d is None]
if miss:
self.calls += len(miss)
results = await asyncio.gather(
*[self._describe_one(client, images[i]) for i in miss]
)
for idx, result in zip(miss, results):
descriptions[idx] = result
if not result.startswith("[vision"):
self.cache.put(keys[idx], result)
return [d or "" for d in descriptions]
async def augment_messages(self, client: httpx.AsyncClient, messages: list) -> list:
if not any(
has_vision_blocks(m.get("content")) for m in messages if isinstance(m, dict)
):
return messages
out: list = []
for m in messages:
content = m.get("content") if isinstance(m, dict) else None
if not has_vision_blocks(content):
out.append(m)
continue
user_text, images = split_text_and_images(content)
descriptions = await self._describe_images(client, images)
vision_block = _format_vision_block(descriptions)
merged = f"{user_text}\n\n{vision_block}" if user_text else vision_block
new_msg = dict(m)
new_msg["content"] = merged
out.append(new_msg)
return out