blindxfish/devplacepy
1
0
Fork 0
forked from retoor/devplacepy
Code Pull Requests Activity

feat: add embeddings endpoint and config for OpenAI-compatible text embeddings via gateway

Browse Source 
Add POST /openai/v1/embeddings route in openai_gateway router, new config fields for embedding upstream URL/model/key/enabled toggle with defaults pointing to OpenRouter Qwen3 8B, INTERNAL_EMBED_MODEL constant in config.py, documentation in docs_api.py and README.md describing the molodetz~embed model mapping, and embed-call tracking in gateway metrics alongside existing chat/vision counters.
This commit is contained in:
retoor
2026-06-14 01:06:18 +00:00
parent 1076696dec
commit c7770ee21a
17 changed files with 662 additions and 62 deletions
Download Patch File Download Diff File Expand all files Collapse all files
devplacepy/services/openai_gateway/config.py
+4
View File
@@ -10,6 +10,9 @@ VISION_URL_DEFAULT = "https://openrouter.ai/api/v1/chat/completions"
VISION_MODEL_DEFAULT = "google/gemma-3-12b-it"
VISION_CACHE_SIZE_DEFAULT = 256
EMBED_URL_DEFAULT = "https://openrouter.ai/api/v1/embeddings"
EMBED_MODEL_DEFAULT = "qwen/qwen3-embedding-8b"
VISION_INSTRUCTION = (
"Describe this image in detail. Note objects, people, scene, any visible "
"text, layout, colors, and anything else that could be relevant for "
@@ -21,6 +24,7 @@ PRICE_CACHE_MISS_PER_M_DEFAULT = 0.14
PRICE_OUTPUT_PER_M_DEFAULT = 0.28
VISION_PRICE_INPUT_PER_M_DEFAULT = 0.0
VISION_PRICE_OUTPUT_PER_M_DEFAULT = 0.0
EMBED_PRICE_INPUT_PER_M_DEFAULT = 0.01
USAGE_RETENTION_HOURS_DEFAULT = 720
devplacepy/services/openai_gateway/gateway.py
+133
View File
@@ -101,6 +101,7 @@ class GatewayRuntime:
self.in_flight = 0
self.peak_in_flight = 0
self.vision_calls = 0
self.embed_calls = 0
self.last_status = 0
self.last_latency_ms = 0
@@ -345,6 +346,137 @@ class GatewayRuntime:
)
return JSONResponse(content=data)
async def handle_embeddings(
self, body: dict, cfg: dict, owner: tuple, user_agent: str, log=None
):
log = log or (lambda message: None)
if not cfg["gateway_embed_enabled"]:
return JSONResponse(
status_code=503,
content={
"error": {
"message": "Embeddings are disabled",
"type": "embeddings_disabled",
}
},
)
client, sem = self._ensure(cfg)
pricing = pricing_from_cfg(cfg)
context_map = parse_context_map(cfg.get("gateway_model_context_map"))
params = extract_params(body)
handle_start = time.monotonic()
requested = body.get("model")
if cfg["gateway_force_model"] or not requested or requested == "molodetz~embed":
model = cfg["gateway_embed_model"]
else:
model = requested
payload = dict(body)
payload["model"] = model
headers = {"Content-Type": "application/json"}
if cfg["gateway_embed_key"]:
headers["Authorization"] = f"Bearer {cfg['gateway_embed_key']}"
else:
log(
"No upstream embeddings API key configured (gateway_embed_key / gateway_vision_key / OPENROUTER_API_KEY); upstream will likely reject the request"
)
resp, exc, timing = await self._send(
client,
sem,
"POST",
cfg["gateway_embed_url"],
headers,
cfg,
log,
json_body=payload,
)
base = {
"owner_kind": owner[0],
"owner_id": owner[1],
"backend": "embed",
"endpoint": "embeddings",
"model": model,
"user_agent": user_agent,
**params,
**timing,
}
def finalize(status_code, success, category, usage=None):
base["total_latency_ms"] = round(
(time.monotonic() - handle_start) * 1000, 3
)
base["gateway_overhead_ms"] = round(
max(
base["total_latency_ms"]
- timing["upstream_latency_ms"]
- timing["queue_wait_ms"],
0.0,
),
3,
)
base["status_code"] = status_code
base["success"] = success
base["error_category"] = category
base["usage"] = usage
self._ledger.record(base, pricing, context_map)
if timing["circuit_open"]:
finalize(503, False, "circuit_open")
return JSONResponse(
status_code=503,
content={
"error": {
"message": "Upstream temporarily unavailable",
"type": "circuit_open",
}
},
)
if exc is not None:
finalize(502, False, classify_error(0, exc))
return JSONResponse(
status_code=502,
content={
"error": {
"message": f"Upstream connection failed: {exc}",
"type": "upstream_error",
}
},
)
if resp.status_code != 200:
finalize(
resp.status_code,
False,
classify_error(resp.status_code, None, resp.text),
)
log(f"embed upstream POST -> {resp.status_code}: {resp.text[:300]}")
return JSONResponse(
status_code=resp.status_code,
content={"error": {"message": resp.text, "type": "upstream_error"}},
)
try:
data = resp.json()
except ValueError:
self.errors += 1
finalize(502, False, "gateway")
log("embed upstream returned 200 but body was not valid JSON")
return JSONResponse(
status_code=502,
content={
"error": {
"message": "invalid upstream response",
"type": "upstream_error",
}
},
)
self.embed_calls += 1
finalize(200, True, None, data.get("usage"))
log(f"embed POST -> 200 ({timing['upstream_latency_ms']:.0f}ms)")
return JSONResponse(content=data)
async def handle_passthrough(
self,
method: str,
@@ -458,6 +590,7 @@ class GatewayRuntime:
"in_flight": self.in_flight,
"peak_in_flight": self.peak_in_flight,
"vision_calls": self.vision_calls,
"embed_calls": self.embed_calls,
"last_status": self.last_status,
"last_latency_ms": self.last_latency_ms,
"pool": self._instances,
devplacepy/services/openai_gateway/service.py
+61
View File
@@ -130,6 +130,38 @@ class GatewayService(BaseService):
help="Image-description LRU cache entries (0 disables caching).",
group="Vision",
),
ConfigField(
"gateway_embed_enabled",
"Embeddings",
type="bool",
default=True,
help="Expose the embeddings model at /openai/v1/embeddings.",
group="Embeddings",
),
ConfigField(
"gateway_embed_url",
"Embeddings URL",
type="url",
default=config.EMBED_URL_DEFAULT,
help="OpenAI-compatible embeddings endpoint requests are forwarded to.",
group="Embeddings",
),
ConfigField(
"gateway_embed_model",
"Embeddings model",
type="str",
default=config.EMBED_MODEL_DEFAULT,
help="Embedding model sent upstream. Clients request it as molodetz~embed.",
group="Embeddings",
),
ConfigField(
"gateway_embed_key",
"Embeddings API key",
type="str",
default="",
help="The key currently in use; falls back to the vision/OPENROUTER key on boot (embeddings default to OpenRouter). Editable.",
group="Embeddings",
),
ConfigField(
"gateway_require_auth",
"Require authentication",
@@ -216,6 +248,15 @@ class GatewayService(BaseService):
minimum=0,
group="Pricing",
),
ConfigField(
"gateway_embed_price_input_per_m",
"Embeddings price input / 1M ($)",
type="float",
default=config.EMBED_PRICE_INPUT_PER_M_DEFAULT,
minimum=0,
help="Fallback only; used when the embeddings upstream returns no native cost.",
group="Pricing",
),
ConfigField(
"gateway_max_retries",
"Max retries",
@@ -290,6 +331,11 @@ class GatewayService(BaseService):
cfg["gateway_vision_key"] = cfg["gateway_vision_key"] or os.environ.get(
"OPENROUTER_API_KEY", ""
)
cfg["gateway_embed_key"] = (
cfg["gateway_embed_key"]
or cfg["gateway_vision_key"]
or os.environ.get("OPENROUTER_API_KEY", "")
)
return cfg
def authorize(self, request: Request) -> bool:
@@ -352,6 +398,18 @@ class GatewayService(BaseService):
self.log("Rejected chat request: JSON body was not an object")
raise HTTPException(status_code=400, detail="Invalid JSON body")
return await runtime.handle_chat(body, cfg, owner, user_agent, self.log)
if subpath == "embeddings" and request.method == "POST":
try:
body = await request.json()
except Exception:
self.log("Rejected embeddings request: invalid JSON body")
raise HTTPException(status_code=400, detail="Invalid JSON body")
if not isinstance(body, dict):
self.log("Rejected embeddings request: JSON body was not an object")
raise HTTPException(status_code=400, detail="Invalid JSON body")
return await runtime.handle_embeddings(
body, cfg, owner, user_agent, self.log
)
body = await request.body()
content_type = request.headers.get("content-type", "")
return await runtime.handle_passthrough(
@@ -395,6 +453,7 @@ class GatewayService(BaseService):
"in_flight": 0,
"peak_in_flight": 0,
"vision_calls": 0,
"embed_calls": 0,
"last_status": 0,
"last_latency_ms": 0,
"pool": 0,
@@ -406,11 +465,13 @@ class GatewayService(BaseService):
{"label": "Requests (lifetime)", "value": m["requests"]},
{"label": "In flight", "value": m["in_flight"]},
{"label": "Vision calls", "value": m["vision_calls"]},
{"label": "Embed calls", "value": m["embed_calls"]},
{"label": "Last status", "value": m["last_status"] or "-"},
{"label": "Last latency", "value": f"{m['last_latency_ms']} ms"},
{"label": "Pool size", "value": m["pool"]},
{"label": "Circuit", "value": "open" if m["circuit_open"] else "closed"},
{"label": "Model", "value": cfg["gateway_model"]},
{"label": "Embed model", "value": cfg["gateway_embed_model"]},
{"label": "Requests 24h", "value": s["requests"]},
{"label": "Success 24h", "value": f"{s['success_pct']}%"},
{"label": "Error rate 24h", "value": f"{s['error_pct']}%"},
devplacepy/services/openai_gateway/usage.py
+10
View File
@@ -35,6 +35,7 @@ class Pricing:
chat_output_per_m: float
vision_input_per_m: float
vision_output_per_m: float
embed_input_per_m: float
def pricing_from_cfg(cfg: dict) -> Pricing:
@@ -64,6 +65,12 @@ def pricing_from_cfg(cfg: dict) -> Pricing:
config.VISION_PRICE_OUTPUT_PER_M_DEFAULT,
)
),
embed_input_per_m=float(
cfg.get(
"gateway_embed_price_input_per_m",
config.EMBED_PRICE_INPUT_PER_M_DEFAULT,
)
),
)
@@ -123,6 +130,9 @@ def compute_cost(
input_cost = norm["prompt"] / PER_MILLION * pricing.vision_input_per_m
output_cost = norm["completion"] / PER_MILLION * pricing.vision_output_per_m
return input_cost + output_cost, input_cost, output_cost, False
if backend == "embed":
input_cost = norm["prompt"] / PER_MILLION * pricing.embed_input_per_m
return input_cost, input_cost, 0.0, False
input_cost = (
norm["cache_hit"] / PER_MILLION * pricing.chat_cache_hit_per_m
+ norm["cache_miss"] / PER_MILLION * pricing.chat_cache_miss_per_m