diff --git a/devplacepy/services/openai_gateway/CLAUDE.md b/devplacepy/services/openai_gateway/CLAUDE.md index d53760a..31da160 100644 --- a/devplacepy/services/openai_gateway/CLAUDE.md +++ b/devplacepy/services/openai_gateway/CLAUDE.md @@ -62,6 +62,10 @@ Do **not** send `reasoning_effort: "none"` to DeepSeek (400: unknown variant). C Vision describe-image calls always disable thinking (they are not a reasoning job). +**Dialect can be overridden when the URL does not identify the provider.** `gateway_thinking_dialect` (Prompt group, `select`, default `auto`, options `auto`/`deepseek`/`openrouter`/`ollama`) forces `thinking.py::thinking_dialect`'s URL-sniffing result - needed for a reverse-proxied Ollama whose URL carries no `:11434`/`ollama` token. `apply_thinking(payload, url, default_enabled, dialect=...)` and the vision augmenter (`VisionAugmenter.vision_dialect`, passed from `cfg["gateway_thinking_dialect"]`) both accept it; `"auto"`/blank falls through to the existing URL-based detection unchanged. + +`thinking.upstream_capabilities(url, dialect="")` derives per-upstream protocol flags (`supports_stream_options`, `supports_stream_usage`, `supports_thinking_field`) from the same effective dialect. Ollama's OpenAI-compatibility layer ignores `stream_options` (and rejects the request outright on some versions), so `handle_chat`'s streaming branch only sends `stream_options: {"include_usage": true}` upstream when `capabilities.supports_stream_options` is true; otherwise a client that asked for `include_usage` gets a logged notice and its stream relayed with no usage chunk. DeepSeek/OpenRouter (and any unrecognised URL) report full support, so this is a no-op for them. + ## Real upstream streaming (`GatewayRuntime._stream_chat_response`) `stream: true` is forwarded to the upstream verbatim (`payload["stream"] = stream`, no more forced `false`) and the connection is opened with `client.send(request, stream=True)` so the response body is read incrementally via `resp.aiter_lines()` instead of buffered whole. Every retry/circuit-breaker mechanic in `_send`/`retry_send` is shared unchanged with the non-streaming path (a 5xx or connection failure before any bytes are forwarded to the client retries exactly as before; `retry_send` now also closes an unread streamed response before retrying, so a retried streaming attempt never leaks the previous connection). Only once the upstream returns `200` does the code path diverge into `_stream_chat_response`. @@ -120,7 +124,7 @@ The gateway is the only place that holds real provider URLs/models/keys. Every o - Defaults live in `config.py`: `INTERNAL_GATEWAY_URL` (`http://localhost:{DEVPLACE_PORT, default 10500}/openai/v1/chat/completions`; the whole base can also be overridden with `DEVPLACE_INTERNAL_BASE_URL`) and `INTERNAL_MODEL` (`molodetz`). `news_ai_url`, `bot_api_url`, `devii_ai_url` default to `INTERNAL_GATEWAY_URL`; their model defaults to `molodetz`. - Each consumer's key falls back to `database.internal_gateway_key()` (reads the `gateway_internal_key` setting) when its own key field/env is unset - the provider-key fallbacks (`DEEPSEEK_API_KEY`/`OPENROUTER_API_KEY`) were removed from news and bots. -- `gateway_force_model` (default on) and a `molodetz`/empty alias in `handle_chat` make the upstream always receive `gateway_model`, so `molodetz` is a stable generic alias. +- `gateway_force_model` (default on) and a `molodetz`/empty alias in `handle_chat` make the upstream always receive `gateway_model`, so `molodetz` is a stable generic alias. **`gateway_allow_client_model`** (Upstream group, bool, default off) lets a client-requested model name through even while `gateway_force_model` is on - useful for an upstream that serves many models by name with no single stable alias (a self-hosted Ollama). The `molodetz` alias is always remapped to `gateway_model` regardless of this flag, so the generic name keeps working for internal callers; only a genuinely named model (`llama3.2`, `qwen3`, ...) bypasses the force. - `database.migrate_ai_gateway_settings()` (called at the end of `init_db()`, under the startup `init_lock`): generates `gateway_internal_key` (uuid4) if missing; migrates `DEEPSEEK_API_KEY`/`OPENROUTER_API_KEY` env into `gateway_api_key`/`gateway_vision_key` when the db value is empty; and rewrites any consumer AI URL still equal to the old `openai.app.molodetz.nl` default to the gateway, plus `bot_model` `deepseek-chat` -> `molodetz` (only uncustomized values). - The gateway's `gateway_api_key`/`gateway_vision_key`/`gateway_internal_key` fields are **non-secret** so the admin services page shows the value actually in use, editable. - Bot LLM calls are synchronous `urllib` but already run via `asyncio.to_thread` (`bot/bot.py`), so the local round-trip never blocks the event loop. diff --git a/devplacepy/services/openai_gateway/gateway.py b/devplacepy/services/openai_gateway/gateway.py index 5ad3456..b96008b 100644 --- a/devplacepy/services/openai_gateway/gateway.py +++ b/devplacepy/services/openai_gateway/gateway.py @@ -20,7 +20,10 @@ from devplacepy.services.openai_gateway.routing import ( resolve_fallback, ) from devplacepy.services.openai_gateway.system_message import apply_system_directives -from devplacepy.services.openai_gateway.thinking import apply_thinking +from devplacepy.services.openai_gateway.thinking import ( + apply_thinking, + upstream_capabilities, +) from devplacepy.services.openai_gateway.usage import ( GatewayUsageLedger, classify_error, @@ -244,6 +247,7 @@ class GatewayRuntime: pricing=pricing, context_map=context_map, app_reference=app_reference, + vision_dialect=cfg.get("gateway_thinking_dialect", "auto"), ) messages = await augmenter.augment_messages(client, messages) self.vision_calls += augmenter.calls @@ -252,10 +256,15 @@ class GatewayRuntime: messages = apply_system_directives(messages, cfg.get("gateway_system_preamble", "")) requested = body.get("model") - if cfg["gateway_force_model"] or not requested or requested == "molodetz": + allow_client_model = bool(cfg.get("gateway_allow_client_model")) + if cfg["gateway_force_model"] and not allow_client_model: + model = cfg["gateway_model"] + elif not requested or requested == "molodetz": model = cfg["gateway_model"] elif overlay is not None: model = requested + elif allow_client_model: + model = requested else: model = cfg["gateway_model"] log(f"requested model {requested!r} has no route, falling back to {model!r}") @@ -275,19 +284,29 @@ class GatewayRuntime: payload["messages"] = messages payload["stream"] = stream if stream: - # Always ask the upstream for usage on its final chunk, regardless of - # whether the client itself requested it, so the ledger always has real - # cost/token numbers for a streamed call; `include_usage` (the client's - # own ask) only controls whether that chunk is relayed to the client. - stream_options = dict(body.get("stream_options") or {}) - stream_options["include_usage"] = True - payload["stream_options"] = stream_options + capabilities = upstream_capabilities( + cfg.get("gateway_upstream_url", ""), + cfg.get("gateway_thinking_dialect", "auto"), + ) + if capabilities.supports_stream_options: + stream_options = dict(body.get("stream_options") or {}) + stream_options["include_usage"] = True + payload["stream_options"] = stream_options + else: + payload.pop("stream_options", None) + if include_usage: + log( + "stream_options.include_usage requested but the upstream " + f"({capabilities.dialect}) does not support it; relaying " + "without a usage chunk" + ) else: payload.pop("stream_options", None) apply_thinking( payload, cfg.get("gateway_upstream_url", ""), default_enabled=bool(cfg.get("gateway_thinking", False)), + dialect=cfg.get("gateway_thinking_dialect", "auto"), ) headers = {"Content-Type": "application/json"} diff --git a/devplacepy/services/openai_gateway/service.py b/devplacepy/services/openai_gateway/service.py index 291181e..e907202 100644 --- a/devplacepy/services/openai_gateway/service.py +++ b/devplacepy/services/openai_gateway/service.py @@ -83,6 +83,19 @@ class GatewayService(BaseService): help="Override the client-requested model with the configured model.", group="Upstream", ), + ConfigField( + "gateway_allow_client_model", + "Allow client-specified model", + type="bool", + default=False, + help="When on, a client-requested model name is honored even though " + "'Force model' is on - except for the molodetz alias, which is always " + "remapped. Useful for upstreams that serve many models by name and " + "have no single stable alias (e.g. a self-hosted Ollama), so a caller " + "can pick llama3.2, qwen3, or whatever is loaded without the gateway " + "silently rewriting it to the configured default.", + group="Upstream", + ), ConfigField( "gateway_api_key", "Upstream API key", @@ -132,6 +145,24 @@ class GatewayService(BaseService): "client disables it.", group="Prompt", ), + ConfigField( + "gateway_thinking_dialect", + "Upstream dialect", + type="select", + default="auto", + options=[ + {"value": "auto", "label": "Auto-detect"}, + {"value": "deepseek", "label": "DeepSeek"}, + {"value": "openrouter", "label": "OpenRouter"}, + {"value": "ollama", "label": "Ollama"}, + ], + help="Protocol dialect the upstream speaks. 'auto' derives it from the " + "upstream URL (deepseek.com / openrouter.ai / :11434 / ollama, defaulting " + "to deepseek for any other URL). Set explicitly when the URL does not " + "identify the provider - e.g. a self-hosted Ollama behind a reverse " + "proxy - so thinking/stream_options are emitted in the right shape.", + group="Prompt", + ), ConfigField( "gateway_vision_enabled", "Vision augmentation", diff --git a/devplacepy/services/openai_gateway/thinking.py b/devplacepy/services/openai_gateway/thinking.py index eb0c6e4..7991167 100644 --- a/devplacepy/services/openai_gateway/thinking.py +++ b/devplacepy/services/openai_gateway/thinking.py @@ -1,5 +1,6 @@ # retoor +from dataclasses import dataclass from typing import Any, Optional CLIENT_THINKING_KEYS = ( @@ -26,6 +27,40 @@ def thinking_dialect(url: str) -> str: return "deepseek" +def _resolve_dialect(url: str, dialect: str) -> str: + effective = dialect.strip().lower() + if effective in ("", "auto"): + return thinking_dialect(url) + return effective + + +@dataclass(frozen=True) +class UpstreamCapabilities: + dialect: str + supports_stream_options: bool + supports_stream_usage: bool + supports_thinking_field: bool + + +def upstream_capabilities( + url: str, dialect: str = "" +) -> UpstreamCapabilities: + effective = _resolve_dialect(url, dialect) + if effective == "ollama": + return UpstreamCapabilities( + dialect=effective, + supports_stream_options=False, + supports_stream_usage=False, + supports_thinking_field=True, + ) + return UpstreamCapabilities( + dialect=effective, + supports_stream_options=True, + supports_stream_usage=True, + supports_thinking_field=True, + ) + + def _as_text(value: Any) -> str: if isinstance(value, str): return value.strip().lower() @@ -118,8 +153,10 @@ def apply_thinking( payload: dict, upstream_url: str, default_enabled: bool = False, + dialect: str = "", ) -> dict: + effective = _resolve_dialect(upstream_url, dialect) specified = client_thinking_enabled(payload) enabled = default_enabled if specified is None else specified - write_thinking(payload, thinking_dialect(upstream_url), enabled) + write_thinking(payload, effective, enabled) return payload diff --git a/devplacepy/services/openai_gateway/vision.py b/devplacepy/services/openai_gateway/vision.py index dd84f14..77e94ec 100644 --- a/devplacepy/services/openai_gateway/vision.py +++ b/devplacepy/services/openai_gateway/vision.py @@ -91,6 +91,7 @@ class VisionAugmenter: cache: VisionCache, referer: str = "", title: str = "", + vision_dialect: str = "", ledger=None, owner: tuple = ("unknown", "unknown"), pricing=None, @@ -103,6 +104,7 @@ class VisionAugmenter: self.cache = cache self.referer = referer self.title = title + self.vision_dialect = vision_dialect self.ledger = ledger self.owner = owner self.pricing = pricing @@ -154,7 +156,7 @@ class VisionAugmenter: "temperature": 0.2, "stream": False, } - apply_thinking(payload, self.vision_url, default_enabled=False) + apply_thinking(payload, self.vision_url, default_enabled=False, dialect=self.vision_dialect) headers = { "Authorization": f"Bearer {self.vision_key}", "Content-Type": "application/json", diff --git a/tests/unit/services/openai_gateway/gateway.py b/tests/unit/services/openai_gateway/gateway.py index 6d64899..58c5941 100644 --- a/tests/unit/services/openai_gateway/gateway.py +++ b/tests/unit/services/openai_gateway/gateway.py @@ -942,6 +942,107 @@ class FakeClientWithUsage_openai_gateway(FakeClient_openai_gateway): ) +def test_ollama_stream_does_not_send_stream_options(local_db, monkeypatch): + monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeClient_openai_gateway) + svc = GatewayService() + cfg = svc.effective_config() + cfg["gateway_upstream_url"] = "http://127.0.0.1:11434/api/chat" + rt = svc.runtime() + run_async( + rt.handle_chat( + { + "messages": [{"role": "user", "content": "hi"}], + "stream": True, + "stream_options": {"include_usage": True}, + }, + cfg, + ("guest", "ollama_stream_opts"), + "test", + "default", + ) + ) + upstream_payload = rt._client.calls[-1][1] + assert upstream_payload["stream"] is True + assert "stream_options" not in upstream_payload + assert upstream_payload["think"] is False + + +def test_ollama_explicit_dialect_overrides_url(local_db, monkeypatch): + monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeClient_openai_gateway) + svc = GatewayService() + cfg = svc.effective_config() + cfg["gateway_upstream_url"] = "https://ai.example.com/v1/chat/completions" + cfg["gateway_thinking_dialect"] = "ollama" + rt = svc.runtime() + run_async( + rt.handle_chat( + {"messages": [{"role": "user", "content": "hi"}]}, + cfg, + ("guest", "ollama_dialect_override"), + "test", + "default", + ) + ) + body = rt._client.calls[-1][1] + assert body["think"] is False + assert "thinking" not in body + + +def test_client_model_allowed_when_allow_client_model_on(local_db, monkeypatch): + monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeClient_openai_gateway) + svc = GatewayService() + cfg = svc.effective_config() + cfg["gateway_force_model"] = True + cfg["gateway_allow_client_model"] = True + cfg["gateway_model"] = "deepseek-chat" + rt = svc.runtime() + run_async( + rt.handle_chat( + {"model": "llama3.2", "messages": [{"role": "user", "content": "hi"}]}, + cfg, + ("guest", "client_model"), + "test", + "default", + ) + ) + assert rt._client.calls[-1][1]["model"] == "llama3.2" + + +def test_molodetz_alias_still_remapped_with_allow_client_model(local_db, monkeypatch): + from devplacepy.services.openai_gateway import routing + + monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeClient_openai_gateway) + original = routing.model_store.get("molodetz") + routing.model_store.remove("molodetz") + try: + svc = GatewayService() + cfg = svc.effective_config() + cfg["gateway_force_model"] = True + cfg["gateway_allow_client_model"] = True + cfg["gateway_model"] = "deepseek-chat" + rt = svc.runtime() + run_async( + rt.handle_chat( + {"model": "molodetz", "messages": [{"role": "user", "content": "hi"}]}, + cfg, + ("guest", "molodetz_alias"), + "test", + "default", + ) + ) + assert rt._client.calls[-1][1]["model"] == "deepseek-chat" + finally: + if original is not None: + routing.model_store.set( + routing.ModelRouteIn( + **{ + field: getattr(original, field) + for field in routing.ModelRouteIn.model_fields + } + ) + ) + + def test_stream_forwarded_to_upstream_with_forced_include_usage(local_db, monkeypatch): monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeClient_openai_gateway) svc = GatewayService() diff --git a/tests/unit/services/openai_gateway/thinking.py b/tests/unit/services/openai_gateway/thinking.py index 2d4e8a7..89cc5cd 100644 --- a/tests/unit/services/openai_gateway/thinking.py +++ b/tests/unit/services/openai_gateway/thinking.py @@ -4,6 +4,7 @@ from devplacepy.services.openai_gateway.thinking import ( apply_thinking, client_thinking_enabled, thinking_dialect, + upstream_capabilities, ) @@ -15,6 +16,38 @@ def test_dialect_from_upstream_url(): assert thinking_dialect("https://unknown.example/v1/chat/completions") == "deepseek" +def test_upstream_capabilities_openai_full_schema(): + caps = upstream_capabilities("https://api.deepseek.com/chat/completions") + assert caps.dialect == "deepseek" + assert caps.supports_stream_options is True + assert caps.supports_stream_usage is True + + +def test_upstream_capabilities_ollama_lacks_stream_options(): + caps = upstream_capabilities("http://127.0.0.1:11434/api/chat") + assert caps.dialect == "ollama" + assert caps.supports_stream_options is False + assert caps.supports_stream_usage is False + + +def test_upstream_capabilities_explicit_dialect_override(): + caps = upstream_capabilities( + "https://ai.example.com/v1/chat/completions", dialect="ollama" + ) + assert caps.dialect == "ollama" + assert caps.supports_stream_options is False + + +def test_apply_thinking_respects_explicit_dialect_override(): + without = apply_thinking({}, "https://ai.example.com/v1/chat/completions") + assert without["thinking"] == {"type": "disabled"} + with_override = apply_thinking( + {}, "https://ai.example.com/v1/chat/completions", dialect="ollama" + ) + assert with_override["think"] is False + assert "thinking" not in with_override + + def test_client_intent_unspecified(): assert client_thinking_enabled({}) is None assert client_thinking_enabled({"messages": []}) is None